From 5a72479951211013a3a2e9e410e7fc2c8cef4934 Mon Sep 17 00:00:00 2001 From: vkukhtyn <37999482+vkukhtyn@users.noreply.github.com> Date: Wed, 10 Mar 2021 14:56:25 +0200 Subject: [PATCH 01/32] [WIP] SNMP initial implementation (#3755) * Add SNMP type to transport profiles * Update reference config for transport profiles * Initial implementation to retrieve a value using SNMP GET message * Fix license header * Move config files * Use async handler for SNMP response * Send SNMP agent response to attributes and telemetry * UI: add SNMP option for device profile * UI: use JSON field to set SNMP profile configuration * Handle device profile update event * Use concurrent hash map for SNMP device sessions * UI: Add device transport configuration control * Cancel async request to avoid memory leak and timeout handling * Start SNMP pooling after application started * Move OID per profile mapping to SNMP transport context * Fix build after merge with 3.2.1-SNAPSHOT * Init device sessions on TB start * Fix build error, refactoring * Update session context on device update * Set device info on session context creating * Refresh pooling params on device or profile update * Update license header * Process device and profile transport config update * Process SNMP response asynchronously * Change polling implementation --- application/pom.xml | 4 + .../src/main/resources/thingsboard.yml | 2 + common/data/pom.xml | 4 + .../common/data/DeviceTransportType.java | 3 +- .../JsonBasedTransportConfiguration.java | 39 ++++ .../data/DeviceTransportConfiguration.java | 3 +- .../SnmpDeviceTransportConfiguration.java | 43 ++++ .../DeviceProfileTransportConfiguration.java | 8 +- .../profile/SnmpDeviceProfileKvMapping.java | 28 +++ .../SnmpProfileTransportConfiguration.java | 44 ++++ common/transport/pom.xml | 1 + common/transport/snmp/pom.xml | 66 ++++++ .../transport/snmp/SnmpTransportContext.java | 142 ++++++++++++ .../transport/snmp/SnmpTransportService.java | 147 +++++++++++++ .../snmp/session/DeviceSessionCtx.java | 206 ++++++++++++++++++ .../snmp/session/SnmpSessionListener.java | 173 +++++++++++++++ .../server/dao/device/DeviceServiceImpl.java | 3 + msa/transport/pom.xml | 1 + msa/transport/snmp/docker/Dockerfile | 33 +++ .../snmp/docker/start-tb-snmp-transport.sh | 33 +++ msa/transport/snmp/pom.xml | 190 ++++++++++++++++ pom.xml | 11 + transport/pom.xml | 1 + transport/snmp/pom.xml | 112 ++++++++++ .../ThingsboardSnmpTransportApplication.java | 48 ++++ .../main/resources/device-data-config.json | 9 + .../main/resources/device-profile-config.json | 21 ++ .../home/components/home-components.module.ts | 3 + ...ile-transport-configuration.component.html | 6 + ...ile-transport-configuration.component.html | 23 ++ ...ofile-transport-configuration.component.ts | 100 +++++++++ ...ice-transport-configuration.component.html | 6 + ...ice-transport-configuration.component.html | 24 ++ ...evice-transport-configuration.component.ts | 100 +++++++++ .../home/pages/device/device.module.ts | 2 + ui-ngx/src/app/shared/models/device.models.ts | 42 +++- .../assets/locale/locale.constant-en_US.json | 9 + 37 files changed, 1681 insertions(+), 9 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java create mode 100644 common/transport/snmp/pom.xml create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java create mode 100644 msa/transport/snmp/docker/Dockerfile create mode 100644 msa/transport/snmp/docker/start-tb-snmp-transport.sh create mode 100644 msa/transport/snmp/pom.xml create mode 100644 transport/snmp/pom.xml create mode 100644 transport/snmp/src/main/java/org/thingsboard/server/snmp/ThingsboardSnmpTransportApplication.java create mode 100644 transport/snmp/src/main/resources/device-data-config.json create mode 100644 transport/snmp/src/main/resources/device-profile-config.json create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts create mode 100644 ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html create mode 100644 ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts diff --git a/application/pom.xml b/application/pom.xml index c1a49b09e3..e3dcd9f0f9 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -85,6 +85,10 @@ org.thingsboard.common.transport coap + + org.thingsboard.common.transport + snmp + org.thingsboard dao diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0cc7c08669..437a672c54 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -566,6 +566,8 @@ transport: bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" bind_port: "${COAP_BIND_PORT:5683}" timeout: "${COAP_TIMEOUT:10000}" + snmp: + enabled: "${SNMP_ENABLED:true}" swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" diff --git a/common/data/pom.xml b/common/data/pom.xml index 4454e628c0..bc977b1c52 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -79,6 +79,10 @@ org.thingsboard protobuf-dynamic + + org.apache.commons + commons-lang3 + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceTransportType.java b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceTransportType.java index 9b9a021fcf..579cdde968 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DeviceTransportType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DeviceTransportType.java @@ -18,5 +18,6 @@ package org.thingsboard.server.common.data; public enum DeviceTransportType { DEFAULT, MQTT, - LWM2M + LWM2M, + SNMP } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java new file mode 100644 index 0000000000..c65e9dd972 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device; + +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; +import com.fasterxml.jackson.annotation.JsonIgnore; + +import java.util.HashMap; +import java.util.Map; + +public abstract class JsonBasedTransportConfiguration { + + @JsonIgnore + private final Map properties = new HashMap<>(); + + @JsonAnyGetter + public Map properties() { + return this.properties; + } + + @JsonAnySetter + public void put(String name, Object value) { + this.properties.put(name, value); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java index 5409dcd339..0ae9f26cf3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java @@ -29,7 +29,8 @@ import org.thingsboard.server.common.data.DeviceTransportType; @JsonSubTypes({ @JsonSubTypes.Type(value = DefaultDeviceTransportConfiguration.class, name = "DEFAULT"), @JsonSubTypes.Type(value = MqttDeviceTransportConfiguration.class, name = "MQTT"), - @JsonSubTypes.Type(value = Lwm2mDeviceTransportConfiguration.class, name = "LWM2M")}) + @JsonSubTypes.Type(value = Lwm2mDeviceTransportConfiguration.class, name = "LWM2M"), + @JsonSubTypes.Type(value = SnmpDeviceTransportConfiguration.class, name = "SNMP")}) public interface DeviceTransportConfiguration { @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java new file mode 100644 index 0000000000..3db31b4f2a --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.data; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.DeviceTransportType; + +@Data +public class SnmpDeviceTransportConfiguration implements DeviceTransportConfiguration { + + private String address; + private int port; + private String community; + private String protocolVersion; + + @Override + public DeviceTransportType getType() { + return DeviceTransportType.SNMP; + } + + @JsonIgnore + public boolean isValid() { + return StringUtils.isNotEmpty(this.address) + && this.port > 0 + && StringUtils.isNotEmpty(this.community) + && StringUtils.isNotEmpty(this.protocolVersion); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java index 2feba45d34..ffbe5f7048 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; @JsonIgnoreProperties(ignoreUnknown = true) @@ -28,9 +27,10 @@ import org.thingsboard.server.common.data.DeviceTransportType; include = JsonTypeInfo.As.PROPERTY, property = "type") @JsonSubTypes({ - @JsonSubTypes.Type(value = DefaultDeviceProfileTransportConfiguration.class, name = "DEFAULT"), - @JsonSubTypes.Type(value = MqttDeviceProfileTransportConfiguration.class, name = "MQTT"), - @JsonSubTypes.Type(value = Lwm2mDeviceProfileTransportConfiguration.class, name = "LWM2M")}) + @JsonSubTypes.Type(value = DefaultDeviceProfileTransportConfiguration.class, name = "DEFAULT"), + @JsonSubTypes.Type(value = MqttDeviceProfileTransportConfiguration.class, name = "MQTT"), + @JsonSubTypes.Type(value = Lwm2mDeviceProfileTransportConfiguration.class, name = "LWM2M"), + @JsonSubTypes.Type(value = SnmpProfileTransportConfiguration.class, name = "SNMP")}) public interface DeviceProfileTransportConfiguration { @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java new file mode 100644 index 0000000000..6eff7cbd34 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.DataType; + +//TODO: rename class +@Data +public class SnmpDeviceProfileKvMapping { + private String key; + private DataType type; + private String method; + private String oid; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java new file mode 100644 index 0000000000..a2529e74e9 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.device.profile; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import org.thingsboard.server.common.data.DeviceTransportType; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Data +public class SnmpProfileTransportConfiguration implements DeviceProfileTransportConfiguration { + + private int poolPeriodMs; + private int timeoutMs; + private int retries; + private List attributes; + private List telemetry; + + @Override + public DeviceTransportType getType() { + return DeviceTransportType.SNMP; + } + + @JsonIgnore + public List getKvMappings() { + return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList()); + } +} diff --git a/common/transport/pom.xml b/common/transport/pom.xml index 957b0d518c..35722cdd1d 100644 --- a/common/transport/pom.xml +++ b/common/transport/pom.xml @@ -39,6 +39,7 @@ mqtt http coap + snmp diff --git a/common/transport/snmp/pom.xml b/common/transport/snmp/pom.xml new file mode 100644 index 0000000000..1f334b9699 --- /dev/null +++ b/common/transport/snmp/pom.xml @@ -0,0 +1,66 @@ + + + 4.0.0 + + + org.thingsboard.common + 3.2.1-SNAPSHOT + transport + + + org.thingsboard.common.transport + snmp + jar + + Thingsboard SNMP Transport Common + https://thingsboard.io + + + UTF-8 + ${basedir}/../../.. + + + + + org.thingsboard.common.transport + transport-api + + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.slf4j + slf4j-api + + + org.snmp4j + snmp4j + + + org.thingsboard.common + dao-api + + + diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java new file mode 100644 index 0000000000..66f852a209 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -0,0 +1,142 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.snmp4j.PDU; +import org.snmp4j.Snmp; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.VariableBinding; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileKvMapping; +import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.transport.TransportContext; +import org.thingsboard.server.dao.device.DeviceCredentialsService; +import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; + +@Service("SnmpTransportContext") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") +@Slf4j +public class SnmpTransportContext extends TransportContext { + @Autowired + DeviceCredentialsService deviceCredentialsService; + + @Autowired + SnmpTransportService snmpTransportService; + + @Getter + private final Map profileTransportConfig = new ConcurrentHashMap<>(); + @Getter + private final Map> pdusPerProfile = new ConcurrentHashMap<>(); + @Getter + private final Map deviceSessions = new ConcurrentHashMap<>(); + + public Optional findAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) { + if (profileTransportConfig.containsKey(deviceProfileId)) { + return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getAttributes()); + } + return Optional.empty(); + } + + public Optional findTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { + if (profileTransportConfig.containsKey(deviceProfileId)) { + return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getTelemetry()); + } + return Optional.empty(); + } + + private Optional findMapping(OID responseOid, List mappings) { + return mappings.stream() + .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) + //TODO: OID shouldn't be duplicated in the config, add backend and UI verification + .findFirst(); + } + + public void initPduListPerProfile() { + profileTransportConfig.forEach(this::updatePduListPerProfile); + } + + public void updatePduListPerProfile(DeviceProfileId id, SnmpProfileTransportConfiguration config) { + pdusPerProfile.put(id, createPduList(config)); + } + + public void updateDeviceSessionCtx(Device device, DeviceProfile deviceProfile, Snmp snmp) { + DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId()); + if (DeviceCredentialsType.ACCESS_TOKEN.equals(credentials.getCredentialsType())) { + SnmpDeviceTransportConfiguration snmpDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + if (snmpDeviceTransportConfiguration.isValid()) { + DeviceSessionCtx deviceSessionCtx = new DeviceSessionCtx(this, credentials.getCredentialsId(), snmpDeviceTransportConfiguration, snmp, device.getId(), deviceProfile); + deviceSessionCtx.createSessionInfo(ctx -> getTransportService().registerAsyncSession(deviceSessionCtx.getSessionInfo(), deviceSessionCtx)); + this.deviceSessions.put(device.getId(), deviceSessionCtx); + } + } else { + log.warn("[{}] Expected credentials type is {} but found {}", device.getId(), DeviceCredentialsType.ACCESS_TOKEN, credentials.getCredentialsType()); + } + } + + public ExecutorService getSnmpCallbackExecutor() { + return snmpTransportService.getSnmpCallbackExecutor(); + } + + private List createPduList(SnmpProfileTransportConfiguration deviceProfileConfig) { + Map> varBindingPerMethod = new HashMap<>(); + + deviceProfileConfig.getKvMappings().forEach(mapping -> varBindingPerMethod + .computeIfAbsent(mapping.getMethod(), v -> new ArrayList<>()) + .add(new VariableBinding(new OID(mapping.getOid())))); + + return varBindingPerMethod.keySet().stream() + .map(method -> { + PDU request = new PDU(); + request.setType(getSnmpMethod(method)); + request.addAll(varBindingPerMethod.get(method)); + return request; + }) + .collect(Collectors.toList()); + } + + //TODO: Extract SNMP methods to enum + private int getSnmpMethod(String configMethod) { + switch (configMethod) { + case "get": + return PDU.GET; + case "getNext": + case "response": + case "set": + default: + return -1; + } + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java new file mode 100644 index 0000000000..5d8e4d84b4 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java @@ -0,0 +1,147 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.snmp4j.Snmp; +import org.snmp4j.transport.DefaultUdpTransportMapping; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageDataIterable; +import org.thingsboard.server.dao.device.DeviceProfileService; +import org.thingsboard.server.dao.device.DeviceService; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.IOException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +@Service("SnmpTransportService") +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") +@Slf4j +public class SnmpTransportService { + + private static final int ENTITY_PACK_LIMIT = 1024; + + @Autowired + private SnmpTransportContext snmpTransportContext; + + @Autowired + DeviceProfileService deviceProfileService; + + @Autowired + TenantService tenantService; + + @Autowired + DeviceService deviceService; + + @Getter + private ExecutorService snmpCallbackExecutor; + private Snmp snmp; + private ScheduledExecutorService pollingExecutor; + + @PostConstruct + public void init() { + log.info("Starting SNMP transport..."); + pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); + //TODO: Set parallelism value in the config + snmpCallbackExecutor = Executors.newWorkStealingPool(20); + initializeSnmp(); + log.info("SNMP transport started!"); + } + + @PreDestroy + public void shutdown() { + log.info("Stopping SNMP transport!"); + if (pollingExecutor != null) { + pollingExecutor.shutdownNow(); + } + if (snmpCallbackExecutor != null) { + snmpCallbackExecutor.shutdownNow(); + } + if (snmp != null) { + try { + snmp.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + log.info("SNMP transport stopped!"); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 2) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting SNMP polling."); + initSessionCtxList(); + startPolling(); + } + + private void initializeSnmp() { + try { + this.snmp = new Snmp(new DefaultUdpTransportMapping()); + this.snmp.listen(); + } catch (IOException e) { + //TODO: what should be done if transport wasn't initialized? + log.error(e.getMessage(), e); + } + } + + private void initSessionCtxList() { + //TODO: This approach works for monolith, in cluster the same data will be fetched by each node. + for (Tenant tenant : new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT)) { + TenantId tenantId = tenant.getTenantId(); + for (DeviceProfile deviceProfile : new PageDataIterable<>(pageLink -> deviceProfileService.findDeviceProfiles(tenantId, pageLink), ENTITY_PACK_LIMIT)) { + if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) { + snmpTransportContext.getProfileTransportConfig().put(deviceProfile.getId(), + (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration()); + initDeviceSessions(deviceProfile); + } + } + } + snmpTransportContext.initPduListPerProfile(); + } + + private void initDeviceSessions(DeviceProfile deviceProfile) { + for (DeviceInfo deviceInfo : new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByTenantIdAndDeviceProfileId(deviceProfile.getTenantId(), deviceProfile.getId(), pageLink), ENTITY_PACK_LIMIT)) { + snmpTransportContext.updateDeviceSessionCtx(deviceInfo, deviceProfile, snmp); + } + } + + private void startPolling() { + //TODO: Get poll period from configuration; + int poolPeriodSeconds = 1; + pollingExecutor.scheduleAtFixedRate(() -> snmpTransportContext.getDeviceSessions().values().forEach(DeviceSessionCtx::executeSnmpRequest), + 0, poolPeriodSeconds, TimeUnit.SECONDS); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java new file mode 100644 index 0000000000..424b964387 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java @@ -0,0 +1,206 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.session; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.snmp4j.CommunityTarget; +import org.snmp4j.Snmp; +import org.snmp4j.Target; +import org.snmp4j.mp.SnmpConstants; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.OctetString; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.auth.SessionInfoCreator; +import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; +import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; + +import java.io.IOException; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +@Slf4j +public class DeviceSessionCtx extends DeviceAwareSessionContext implements SessionMsgListener { + private final AtomicInteger msgIdSeq = new AtomicInteger(0); + + @Getter + @Setter + private SnmpDeviceTransportConfiguration deviceTransportConfig; + @Getter + @Setter + private SnmpSessionListener snmpSessionListener; + @Getter + @Setter + private Target target; + @Getter + @Setter + private volatile TransportProtos.SessionInfoProto sessionInfo; + + private Snmp snmp; + private SnmpProfileTransportConfiguration snmpProfileTransportConfiguration; + private long previousRequestExecutedAt = 0; + + public DeviceSessionCtx(SnmpTransportContext transportContext, String token, SnmpDeviceTransportConfiguration deviceTransportConfig, + Snmp snmp, DeviceId deviceId, DeviceProfile deviceProfile) { + super(UUID.randomUUID()); + this.snmpSessionListener = new SnmpSessionListener(transportContext, token); + super.setDeviceId(deviceId); + super.setDeviceProfile(deviceProfile); + //TODO: What should be done if snmp null? + if (snmp != null) { + this.snmp = snmp; + } + this.snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + initTarget(this.snmpProfileTransportConfiguration, deviceTransportConfig); + } + + @Override + public int nextMsgId() { + return msgIdSeq.incrementAndGet(); + } + + @Override + public void onGetAttributesResponse(GetAttributeResponseMsg getAttributesResponse) { + } + + @Override + public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { + } + + @Override + public void onRemoteSessionCloseCommand(SessionCloseNotificationProto sessionCloseNotification) { + } + + @Override + public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) { + } + + @Override + public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) { + } + + @Override + public void onDeviceProfileUpdate(TransportProtos.SessionInfoProto newSessionInfo, DeviceProfile deviceProfile) { + super.onDeviceProfileUpdate(sessionInfo, deviceProfile); + if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) { + snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + snmpSessionListener.getSnmpTransportContext().getProfileTransportConfig().put( + deviceProfile.getId(), + snmpProfileTransportConfiguration); + snmpSessionListener.getSnmpTransportContext().updatePduListPerProfile(deviceProfile.getId(), snmpProfileTransportConfiguration); + } else { + //TODO: should the context be removed from the map? + } + } + + @Override + public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { + super.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); + if (super.getDeviceProfile() != null && DeviceTransportType.SNMP.equals(super.getDeviceProfile().getTransportType())) { + snmpSessionListener.getSnmpTransportContext().updateDeviceSessionCtx(device, deviceProfile, null); + SnmpProfileTransportConfiguration profileTransportConfig = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + SnmpDeviceTransportConfiguration deviceTransportConfig = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + initTarget(profileTransportConfig, deviceTransportConfig); + } else { + //TODO: should the context be removed from the map? + } + } + + public void createSessionInfo(Consumer registerSession) { + getSnmpSessionListener().getSnmpTransportContext().getTransportService().process(DeviceTransportType.SNMP, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(getSnmpSessionListener().getToken()).build(), + new TransportServiceCallback() { + @Override + public void onSuccess(ValidateDeviceCredentialsResponse msg) { + if (msg.hasDeviceInfo()) { + sessionInfo = SessionInfoCreator.create(msg, getSnmpSessionListener().getSnmpTransportContext(), UUID.randomUUID()); + registerSession.accept(sessionInfo); + setDeviceInfo(msg.getDeviceInfo()); + } else { + log.warn("[{}] Failed to process device auth", getDeviceId()); + } + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to process device auth", getDeviceId(), e); + } + }); + } + + public void executeSnmpRequest() { + long timeNow = System.currentTimeMillis(); + long nextRequestExecutionTime = previousRequestExecutedAt + snmpProfileTransportConfiguration.getPoolPeriodMs(); + if (nextRequestExecutionTime < timeNow) { + previousRequestExecutedAt = timeNow; + + snmpSessionListener.getSnmpTransportContext().getPdusPerProfile().get(deviceProfile.getId()).forEach(pdu -> { + try { + log.debug("[{}] Sending SNMP message...", pdu.getRequestID()); + snmp.send(pdu, + target, + deviceProfile.getId(), + snmpSessionListener); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + }); + } + } + + private void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + this.deviceTransportConfig = deviceTransportConfig; + CommunityTarget communityTarget = new CommunityTarget(); + communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + this.deviceTransportConfig.getAddress() + "/" + this.deviceTransportConfig.getPort())); + communityTarget.setVersion(getSnmpVersion(this.deviceTransportConfig.getProtocolVersion())); + communityTarget.setCommunity(new OctetString(this.deviceTransportConfig.getCommunity())); + communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); + communityTarget.setRetries(profileTransportConfig.getRetries()); + this.target = communityTarget; + log.info("SNMP target initialized: {}", this.target); + } + + //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer) + private int getSnmpVersion(String configSnmpVersion) { + switch (configSnmpVersion) { + case ("v1"): + return SnmpConstants.version1; + case ("v2c"): + return SnmpConstants.version2c; + case ("v3"): + return SnmpConstants.version3; + default: + return -1; + } + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java new file mode 100644 index 0000000000..35aa1a9347 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java @@ -0,0 +1,173 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.session; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.snmp4j.PDU; +import org.snmp4j.Snmp; +import org.snmp4j.event.ResponseEvent; +import org.snmp4j.event.ResponseListener; +import org.snmp4j.smi.VariableBinding; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.transport.TransportContext; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.adaptor.AdaptorException; +import org.thingsboard.server.common.transport.adaptor.JsonConverter; +import org.thingsboard.server.common.transport.auth.SessionInfoCreator; +import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; + +import java.util.UUID; +import java.util.function.Consumer; + +@Slf4j +@AllArgsConstructor +public class SnmpSessionListener implements ResponseListener { + + @Getter + private final SnmpTransportContext snmpTransportContext; + + @Getter + private final String token; + + @Override + public void onResponse(ResponseEvent event) { + ((Snmp) event.getSource()).cancel(event.getRequest(), this); + snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(event)); + } + + private void processSnmpResponse(ResponseEvent event) { + PDU response = event.getResponse(); + if (event.getError() != null) { + log.warn("Response error: {}", event.getError().getMessage(), event.getError()); + } + + if (response != null) { + log.debug("[{}] Processing SNMP response: {}", response.getRequestID(), response); + + DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); + TransportService transportService = snmpTransportContext.getTransportService(); + for (int i = 0; i < response.size(); i++) { + VariableBinding vb = response.get(i); + snmpTransportContext.findAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(), + new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { + try { + transportService.process(sessionInfo, + convertToPostAttributes(kvMapping.getKey(), kvMapping.getType(), vb.toValueString()), + TransportServiceCallback.EMPTY); + reportActivity(sessionInfo); + } catch (Exception e) { + log.warn("Failed to process SNMP response: {}", e.getMessage(), e); + } + }))); + snmpTransportContext.findTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(), + new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { + try { + transportService.process(sessionInfo, + convertToPostTelemetry(kvMapping.getKey(), kvMapping.getType(), vb.toValueString()), + TransportServiceCallback.EMPTY); + reportActivity(sessionInfo); + + } catch (Exception e) { + log.warn("Failed to process SNMP response: {}", e.getMessage(), e); + } + }))); + } + } else { + log.warn("No SNMP response, requestId: {}", event.getRequest().getRequestID()); + } + } + + private TransportProtos.PostAttributeMsg convertToPostAttributes(String keyName, DataType dataType, String payload) throws AdaptorException { + try { + return JsonConverter.convertToAttributesProto(getKvJson(keyName, dataType, payload)); + } catch (IllegalStateException | JsonSyntaxException ex) { + //TODO: change the exception type + throw new AdaptorException(ex); + } + } + + private TransportProtos.PostTelemetryMsg convertToPostTelemetry(String keyName, DataType dataType, String payload) throws AdaptorException { + try { + return JsonConverter.convertToTelemetryProto(getKvJson(keyName, dataType, payload)); + } catch (IllegalStateException | JsonSyntaxException ex) { + //TODO: change the exception type + throw new AdaptorException(ex); + } + } + + private JsonElement getKvJson(String keyName, DataType dataType, String payload) throws AdaptorException { + JsonObject result = new JsonObject(); + switch (dataType) { + case LONG: + result.addProperty(keyName, Long.parseLong(payload)); + break; + case BOOLEAN: + result.addProperty(keyName, Boolean.parseBoolean(payload)); + break; + case DOUBLE: + result.addProperty(keyName, Double.parseDouble(payload)); + break; + case STRING: + result.addProperty(keyName, payload); + break; + default: + //TODO: change the exception type + throw new AdaptorException("Unsupported data type"); + } + return new JsonParser().parse(result.toString()); + } + + private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { + snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(false) + .setRpcSubscription(false) + .setLastActivityTime(System.currentTimeMillis()) + .build(), TransportServiceCallback.EMPTY); + } + + @AllArgsConstructor + private static class DeviceAuthCallback implements TransportServiceCallback { + private final TransportContext transportContext; + private final Consumer onSuccess; + + @Override + public void onSuccess(ValidateDeviceCredentialsResponse msg) { + if (msg.hasDeviceInfo()) { + onSuccess.accept(SessionInfoCreator.create(msg, transportContext, UUID.randomUUID())); + } else { + log.warn("Failed to process device auth"); + } + } + + @Override + public void onError(Throwable e) { + log.warn("Failed to process device auth", e); + } + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index cd161d650e..c088d18d3c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -47,6 +47,7 @@ import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConf import org.thingsboard.server.common.data.device.data.DeviceData; import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.MqttDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -240,6 +241,8 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe case LWM2M: deviceData.setTransportConfiguration(new Lwm2mDeviceTransportConfiguration()); break; + case SNMP: + deviceData.setTransportConfiguration(new SnmpDeviceTransportConfiguration()); } } return deviceData; diff --git a/msa/transport/pom.xml b/msa/transport/pom.xml index 6fdedfbfcf..6a9fe17fc2 100644 --- a/msa/transport/pom.xml +++ b/msa/transport/pom.xml @@ -38,6 +38,7 @@ mqtt http coap + snmp diff --git a/msa/transport/snmp/docker/Dockerfile b/msa/transport/snmp/docker/Dockerfile new file mode 100644 index 0000000000..e39624d538 --- /dev/null +++ b/msa/transport/snmp/docker/Dockerfile @@ -0,0 +1,33 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk8 + +COPY start-tb-snmp-transport.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-snmp-transport.sh /usr/bin + +RUN yes | dpkg -i /tmp/${pkg.name}.deb +RUN rm /tmp/${pkg.name}.deb + +RUN systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || : + +RUN chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-snmp-transport.sh"] diff --git a/msa/transport/snmp/docker/start-tb-snmp-transport.sh b/msa/transport/snmp/docker/start-tb-snmp-transport.sh new file mode 100644 index 0000000000..2e69cf26f9 --- /dev/null +++ b/msa/transport/snmp/docker/start-tb-snmp-transport.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER="/config" +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +export LOADER_PATH=/config,${LOADER_PATH} + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.snmp.ThingsboardSnmpTransportApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=/config/logback.xml \ + org.springframework.boot.loader.PropertiesLauncher diff --git a/msa/transport/snmp/pom.xml b/msa/transport/snmp/pom.xml new file mode 100644 index 0000000000..df5caa658c --- /dev/null +++ b/msa/transport/snmp/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard.msa + transport + 3.2.1-SNAPSHOT + + + org.thingsboard.msa.transport + snmp + pom + + ThingsBoard SNMP Transport Microservice + https://thingsboard.io + ThingsBoard SNMP Transport Microservice + + + UTF-8 + ${basedir}/../../.. + tb-snmp-transport + tb-snmp-transport + /var/log/${pkg.name} + /usr/share/${pkg.name} + + + + + org.thingsboard.transport + snmp + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-tb-snmp-transport-deb + package + + copy + + + + + org.thingsboard.transport + snmp + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/pom.xml b/pom.xml index 4ec162be8f..063cb027cb 100755 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,7 @@ 1.0.2TB 3.4.0 7.54.2 + 2.8.5 @@ -838,6 +839,11 @@ coap ${project.version} + + org.thingsboard.common.transport + snmp + ${project.version} + org.thingsboard dao @@ -1408,6 +1414,11 @@ + + org.snmp4j + snmp4j + ${snmp4j.version} + diff --git a/transport/pom.xml b/transport/pom.xml index bcafb9a184..98429f2329 100644 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -37,6 +37,7 @@ http mqtt coap + snmp diff --git a/transport/snmp/pom.xml b/transport/snmp/pom.xml new file mode 100644 index 0000000000..fbcca16149 --- /dev/null +++ b/transport/snmp/pom.xml @@ -0,0 +1,112 @@ + + + 4.0.0 + + + org.thingsboard + 3.2.1-SNAPSHOT + transport + + + org.thingsboard.transport + snmp + jar + + Thingsboard SNMP Transport Service + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + java + false + process-resources + package + tb-snmp-transport + false + ${project.build.directory}/windows + ThingsBoard SNMP Transport Service + org.thingsboard.server.snmp.ThingsboardSnmpTransportApplication + + + + + org.thingsboard.common.transport + snmp + + + org.thingsboard.common + queue + + + org.springframework.boot + spring-boot-starter-web + + + + + ${pkg.name}-${project.version} + + + ${project.basedir}/src/main/resources + + + + + org.apache.maven.plugins + maven-resources-plugin + + + org.apache.maven.plugins + maven-dependency-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + org.springframework.boot + spring-boot-maven-plugin + + + org.thingsboard + gradle-maven-plugin + + + org.apache.maven.plugins + maven-assembly-plugin + + + org.apache.maven.plugins + maven-install-plugin + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/transport/snmp/src/main/java/org/thingsboard/server/snmp/ThingsboardSnmpTransportApplication.java b/transport/snmp/src/main/java/org/thingsboard/server/snmp/ThingsboardSnmpTransportApplication.java new file mode 100644 index 0000000000..47d96db8c5 --- /dev/null +++ b/transport/snmp/src/main/java/org/thingsboard/server/snmp/ThingsboardSnmpTransportApplication.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.snmp; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.util.Arrays; + +@SpringBootConfiguration +@EnableAsync +@EnableScheduling +@ComponentScan({"org.thingsboard.server.snmp", "org.thingsboard.server.common", "org.thingsboard.server.transport.snmp", "org.thingsboard.server.queue"}) +public class ThingsboardSnmpTransportApplication { + + private static final String SPRING_CONFIG_NAME_KEY = "--spring.config.name"; + private static final String DEFAULT_SPRING_CONFIG_PARAM = SPRING_CONFIG_NAME_KEY + "=" + "tb-snmp-transport"; + + public static void main(String[] args) { + SpringApplication.run(ThingsboardSnmpTransportApplication.class, updateArguments(args)); + } + + private static String[] updateArguments(String[] args) { + if (Arrays.stream(args).noneMatch(arg -> arg.startsWith(SPRING_CONFIG_NAME_KEY))) { + String[] modifiedArgs = new String[args.length + 1]; + System.arraycopy(args, 0, modifiedArgs, 0, args.length); + modifiedArgs[args.length] = DEFAULT_SPRING_CONFIG_PARAM; + return modifiedArgs; + } + return args; + } +} diff --git a/transport/snmp/src/main/resources/device-data-config.json b/transport/snmp/src/main/resources/device-data-config.json new file mode 100644 index 0000000000..e425ccc3a6 --- /dev/null +++ b/transport/snmp/src/main/resources/device-data-config.json @@ -0,0 +1,9 @@ +{ + "deviceName": "Thermostat T1", + "snmpConfig": { + "address": "192.168.1.2", + "port": 161, + "community": "U5J=$HWj6f@7", + "protocolVersion": "v2c" + } +} diff --git a/transport/snmp/src/main/resources/device-profile-config.json b/transport/snmp/src/main/resources/device-profile-config.json new file mode 100644 index 0000000000..b272913045 --- /dev/null +++ b/transport/snmp/src/main/resources/device-profile-config.json @@ -0,0 +1,21 @@ +{ + "poolPeriodMs": 10000, + "timeoutMs": 5000, + "retries": 5, + "attributes": [ + { + "key": "snmpNodeManagerEmail", + "type": "STRING", + "method": "get", + "oid": ".1.3.6.1.2.1.1.4.0" + } + ], + "telemetry": [ + { + "key": "snmpNodeSysUpTime", + "type": "LONG", + "method": "get", + "oid": ".1.3.6.1.2.1.1.3.0" + } + ] +} \ No newline at end of file diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index 3552821a8b..abcbb5989f 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -97,6 +97,7 @@ import { DeviceProfileDialogComponent } from '@home/components/profile/device-pr import { DeviceProfileAutocompleteComponent } from '@home/components/profile/device-profile-autocomplete.component'; import { MqttDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/mqtt-device-profile-transport-configuration.component'; import { Lwm2mDeviceProfileTransportConfigurationComponent } from '@home/components/profile/device/lwm2m-device-profile-transport-configuration.component'; +import { SnmpDeviceProfileTransportConfigurationComponent } from './profile/device/snmp-device-profile-transport-configuration.component'; import { DeviceProfileAlarmsComponent } from '@home/components/profile/alarm/device-profile-alarms.component'; import { DeviceProfileAlarmComponent } from '@home/components/profile/alarm/device-profile-alarm.component'; import { CreateAlarmRulesComponent } from '@home/components/profile/alarm/create-alarm-rules.component'; @@ -199,6 +200,7 @@ import { CopyDeviceCredentialsComponent } from '@home/components/device/copy-dev DefaultDeviceProfileTransportConfigurationComponent, MqttDeviceProfileTransportConfigurationComponent, Lwm2mDeviceProfileTransportConfigurationComponent, + SnmpDeviceProfileTransportConfigurationComponent, DeviceProfileTransportConfigurationComponent, CreateAlarmRulesComponent, AlarmRuleComponent, @@ -288,6 +290,7 @@ import { CopyDeviceCredentialsComponent } from '@home/components/device/copy-dev DefaultDeviceProfileTransportConfigurationComponent, MqttDeviceProfileTransportConfigurationComponent, Lwm2mDeviceProfileTransportConfigurationComponent, + SnmpDeviceProfileTransportConfigurationComponent, DeviceProfileTransportConfigurationComponent, CreateAlarmRulesComponent, AlarmRuleComponent, diff --git a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.html index 78707a9221..b877ef0015 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.html +++ b/ui-ngx/src/app/modules/home/components/profile/device/device-profile-transport-configuration.component.html @@ -35,5 +35,11 @@ formControlName="configuration"> + + + + diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html b/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html new file mode 100644 index 0000000000..f57b1ac716 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.html @@ -0,0 +1,23 @@ + +
+ + +
diff --git a/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts new file mode 100644 index 0000000000..96f7454cba --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/profile/device/snmp-device-profile-transport-configuration.component.ts @@ -0,0 +1,100 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import {Component, forwardRef, Input, OnInit} from '@angular/core'; +import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; +import {Store} from '@ngrx/store'; +import {AppState} from '@app/core/core.state'; +import {coerceBooleanProperty} from '@angular/cdk/coercion'; +import { + DeviceProfileTransportConfiguration, + DeviceTransportType, + SnmpDeviceProfileTransportConfiguration +} from '@shared/models/device.models'; +import {isDefinedAndNotNull} from "@core/utils"; + +export interface OidMappingConfiguration { + isAttribute: boolean; + key: string; + type: string; + method: string; + oid: string; +} + +@Component({ + selector: 'tb-snmp-device-profile-transport-configuration', + templateUrl: './snmp-device-profile-transport-configuration.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceProfileTransportConfigurationComponent), + multi: true + }] +}) +export class SnmpDeviceProfileTransportConfigurationComponent implements ControlValueAccessor, OnInit { + snmpDeviceProfileTransportConfigurationFormGroup: FormGroup; + private requiredValue: boolean; + private configuration = []; + + get required(): boolean { + return this.requiredValue; + } + + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + private propagateChange = (v: any) => { + } + + constructor(private store: Store, private fb: FormBuilder) { + } + + ngOnInit(): void { + this.snmpDeviceProfileTransportConfigurationFormGroup = this.fb.group({ + configuration: [null, Validators.required] + }); + this.snmpDeviceProfileTransportConfigurationFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + writeValue(value: SnmpDeviceProfileTransportConfiguration | null): void { + if (isDefinedAndNotNull(value)) { + this.snmpDeviceProfileTransportConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + } + } + + private updateModel() { + let configuration: DeviceProfileTransportConfiguration = null; + if (this.snmpDeviceProfileTransportConfigurationFormGroup.valid) { + configuration = this.snmpDeviceProfileTransportConfigurationFormGroup.getRawValue().configuration; + configuration.type = DeviceTransportType.SNMP; + } + this.propagateChange(configuration); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.html b/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.html index 73cceba5bc..02799a840c 100644 --- a/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.html +++ b/ui-ngx/src/app/modules/home/pages/device/data/device-transport-configuration.component.html @@ -35,5 +35,11 @@ formControlName="configuration"> + + + + diff --git a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html new file mode 100644 index 0000000000..fc9f615db9 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.html @@ -0,0 +1,24 @@ + +
+ + +
diff --git a/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts new file mode 100644 index 0000000000..165a2906f4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/pages/device/data/snmp-device-transport-configuration.component.ts @@ -0,0 +1,100 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import {Component, forwardRef, Input, OnInit} from '@angular/core'; +import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; +import {Store} from '@ngrx/store'; +import {AppState} from '@app/core/core.state'; +import {coerceBooleanProperty} from '@angular/cdk/coercion'; +import { + DeviceTransportConfiguration, + DeviceTransportType, + SnmpDeviceTransportConfiguration +} from '@shared/models/device.models'; + +@Component({ + selector: 'tb-snmp-device-transport-configuration', + templateUrl: './snmp-device-transport-configuration.component.html', + styleUrls: [], + providers: [{ + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SnmpDeviceTransportConfigurationComponent), + multi: true + }] +}) +export class SnmpDeviceTransportConfigurationComponent implements ControlValueAccessor, OnInit { + + snmpDeviceTransportConfigurationFormGroup: FormGroup; + + private requiredValue: boolean; + + get required(): boolean { + return this.requiredValue; + } + + @Input() + set required(value: boolean) { + this.requiredValue = coerceBooleanProperty(value); + } + + @Input() + disabled: boolean; + + private propagateChange = (v: any) => { + }; + + constructor(private store: Store, + private fb: FormBuilder) { + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + ngOnInit() { + this.snmpDeviceTransportConfigurationFormGroup = this.fb.group({ + configuration: [null, Validators.required] + }); + this.snmpDeviceTransportConfigurationFormGroup.valueChanges.subscribe(() => { + this.updateModel(); + }); + } + + setDisabledState(isDisabled: boolean): void { + this.disabled = isDisabled; + if (this.disabled) { + this.snmpDeviceTransportConfigurationFormGroup.disable({emitEvent: false}); + } else { + this.snmpDeviceTransportConfigurationFormGroup.enable({emitEvent: false}); + } + } + + writeValue(value: SnmpDeviceTransportConfiguration | null): void { + this.snmpDeviceTransportConfigurationFormGroup.patchValue({configuration: value}, {emitEvent: false}); + } + + private updateModel() { + let configuration: DeviceTransportConfiguration = null; + if (this.snmpDeviceTransportConfigurationFormGroup.valid) { + configuration = this.snmpDeviceTransportConfigurationFormGroup.getRawValue().configuration; + configuration.type = DeviceTransportType.SNMP; + } + this.propagateChange(configuration); + } +} diff --git a/ui-ngx/src/app/modules/home/pages/device/device.module.ts b/ui-ngx/src/app/modules/home/pages/device/device.module.ts index 7ba9a5c0fe..9347fab505 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device.module.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device.module.ts @@ -31,6 +31,7 @@ import { DefaultDeviceTransportConfigurationComponent } from './data/default-dev import { DeviceTransportConfigurationComponent } from './data/device-transport-configuration.component'; import { MqttDeviceTransportConfigurationComponent } from './data/mqtt-device-transport-configuration.component'; import { Lwm2mDeviceTransportConfigurationComponent } from './data/lwm2m-device-transport-configuration.component'; +import { SnmpDeviceTransportConfigurationComponent } from './data/snmp-device-transport-configuration.component'; @NgModule({ declarations: [ @@ -39,6 +40,7 @@ import { Lwm2mDeviceTransportConfigurationComponent } from './data/lwm2m-device- DefaultDeviceTransportConfigurationComponent, MqttDeviceTransportConfigurationComponent, Lwm2mDeviceTransportConfigurationComponent, + SnmpDeviceTransportConfigurationComponent, DeviceTransportConfigurationComponent, DeviceDataComponent, DeviceComponent, diff --git a/ui-ngx/src/app/shared/models/device.models.ts b/ui-ngx/src/app/shared/models/device.models.ts index dcf3f34093..fd293aae64 100644 --- a/ui-ngx/src/app/shared/models/device.models.ts +++ b/ui-ngx/src/app/shared/models/device.models.ts @@ -29,13 +29,15 @@ import * as _moment from 'moment'; import { AbstractControl, ValidationErrors } from '@angular/forms'; export enum DeviceProfileType { - DEFAULT = 'DEFAULT' + DEFAULT = 'DEFAULT', + SNMP = 'SNMP' } export enum DeviceTransportType { DEFAULT = 'DEFAULT', MQTT = 'MQTT', // LWM2M = 'LWM2M' + SNMP = 'SNMP' } export enum MqttTransportPayloadType { @@ -68,6 +70,13 @@ export const deviceProfileTypeConfigurationInfoMap = new Map( [DeviceTransportType.DEFAULT, 'device-profile.transport-type-default-hint'], [DeviceTransportType.MQTT, 'device-profile.transport-type-mqtt-hint'], // [DeviceTransportType.LWM2M, 'device-profile.transport-type-lwm2m-hint'] + [DeviceTransportType.SNMP, 'device-profile.transport-type-snmp-hint'], ] ); @@ -128,6 +139,13 @@ export const deviceTransportTypeConfigurationInfoMap = new Map[+] and multi-level [#] wildcards supported.", "telemetry-topic-filter": "Telemetry topic filter", From 37b88908fd12bb80f5c2a3756feefa009bd9e8f9 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 10 Mar 2021 15:25:35 +0200 Subject: [PATCH 02/32] Fix merge errors --- common/transport/snmp/pom.xml | 2 +- msa/transport/snmp/pom.xml | 2 +- transport/snmp/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/transport/snmp/pom.xml b/common/transport/snmp/pom.xml index 1f334b9699..c7e22ecc80 100644 --- a/common/transport/snmp/pom.xml +++ b/common/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard.common - 3.2.1-SNAPSHOT + 3.3.0-SNAPSHOT transport diff --git a/msa/transport/snmp/pom.xml b/msa/transport/snmp/pom.xml index df5caa658c..62448d8763 100644 --- a/msa/transport/snmp/pom.xml +++ b/msa/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard.msa transport - 3.2.1-SNAPSHOT + 3.3.0-SNAPSHOT org.thingsboard.msa.transport diff --git a/transport/snmp/pom.xml b/transport/snmp/pom.xml index fbcca16149..ed446a5612 100644 --- a/transport/snmp/pom.xml +++ b/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 3.2.1-SNAPSHOT + 3.3.0-SNAPSHOT transport From 8f2438d6ab57ca22b940a001e9a05e68225e6c69 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 22 Mar 2021 17:17:42 +0200 Subject: [PATCH 03/32] SNMP devices balancing (#4254) * Fix merge errors * Implement SNMP transports balancing * Refactor; implement transport device cache * Refactor * Finish up device lifecycle handling implementing; refactor * Refactor * Change base image to thingsboard/openjdk11 for msa snmp transport * Refactor * Change transport services names to upper-case --- .../actors/service/DefaultActorService.java | 4 +- .../DefaultTbApiUsageStateService.java | 2 +- .../apiusage/TbApiUsageStateService.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../DefaultTbRuleEngineConsumerService.java | 2 +- .../service/queue/TbCoreConsumerService.java | 2 +- .../queue/TbRuleEngineConsumerService.java | 2 +- .../processing/AbstractConsumerService.java | 2 +- .../state/DefaultDeviceStateService.java | 2 +- .../service/state/DeviceStateService.java | 2 +- .../DefaultSubscriptionManagerService.java | 2 +- .../DefaultTbLocalSubscriptionService.java | 5 +- .../SubscriptionManagerService.java | 2 +- .../TbLocalSubscriptionService.java | 4 +- .../AbstractSubscriptionService.java | 19 +- .../telemetry/AlarmSubscriptionService.java | 3 +- .../DefaultAlarmSubscriptionService.java | 23 - .../TelemetrySubscriptionService.java | 3 +- .../transport/DefaultTransportApiService.java | 94 ++++- .../server/dao/device/DeviceService.java | 3 + .../common/data/TbTransportService.java | 20 + .../data/DeviceTransportConfiguration.java | 4 +- .../SnmpProfileTransportConfiguration.java | 10 +- .../DefaultTbServiceInfoProvider.java | 20 + .../queue/discovery/HashPartitionService.java | 20 +- .../queue/discovery/PartitionService.java | 4 + .../discovery/TbApplicationEventListener.java | 1 + .../queue/discovery/ZkDiscoveryService.java | 12 +- .../ClusterTopologyChangeEvent.java | 3 +- .../{ => event}/PartitionChangeEvent.java | 3 +- .../event/ServiceListChangedEvent.java | 35 ++ .../{ => event}/TbApplicationEvent.java | 2 +- .../queue/util/TbSnmpTransportComponent.java | 29 ++ common/queue/src/main/proto/queue.proto | 35 ++ .../transport/coap/CoapTransportService.java | 9 +- .../transport/http/DeviceApiController.java | 10 +- .../lwm2m/server/LwM2mTransportService.java | 3 +- .../server/LwM2mTransportServiceImpl.java | 6 + .../transport/mqtt/MqttTransportService.java | 8 +- common/transport/snmp/pom.xml | 4 - .../transport/snmp/SnmpDeviceSimulator.java | 81 ++++ .../transport/snmp/SnmpTransportContext.java | 311 +++++++++++--- .../transport/snmp/SnmpTransportService.java | 147 ------- .../ServiceListChangedEventListener.java | 35 ++ .../event/SnmpTransportListChangedEvent.java | 24 ++ ...SnmpTransportListChangedEventListener.java | 34 ++ .../service/ProtoTransportEntityService.java | 92 ++++ .../SnmpTransportBalancingService.java | 92 ++++ .../SnmpTransportService.java} | 149 +++++-- .../snmp/session/DeviceSessionContext.java | 166 ++++++++ .../snmp/session/DeviceSessionCtx.java | 206 --------- .../common/transport/DeviceUpdatedEvent.java | 28 ++ .../common/transport/SessionMsgListener.java | 4 + .../common/transport/TransportContext.java | 1 + .../common/transport/TransportService.java | 12 + .../DefaultTransportDeviceProfileCache.java | 4 +- .../service/DefaultTransportService.java | 77 +++- .../server/dao/device/DeviceDao.java | 4 + .../server/dao/device/DeviceServiceImpl.java | 7 + .../dao/sql/device/DeviceRepository.java | 6 + .../server/dao/sql/device/JpaDeviceDao.java | 6 + docker/tb-transports/snmp/conf/logback.xml | 50 +++ .../snmp/conf/tb-snmp-transport.conf | 23 + msa/transport/snmp/docker/Dockerfile | 2 +- transport/snmp/src/main/conf/logback.xml | 45 ++ .../snmp/src/main/conf/tb-snmp-transport.conf | 22 + transport/snmp/src/main/resources/logback.xml | 36 ++ .../src/main/resources/tb-snmp-transport.yml | 394 ++++++++++++++++++ 68 files changed, 1923 insertions(+), 553 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java rename common/queue/src/main/java/org/thingsboard/server/queue/discovery/{ => event}/ClusterTopologyChangeEvent.java (91%) rename common/queue/src/main/java/org/thingsboard/server/queue/discovery/{ => event}/PartitionChangeEvent.java (93%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java rename common/queue/src/main/java/org/thingsboard/server/queue/discovery/{ => event}/TbApplicationEvent.java (95%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java delete mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java rename common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/{session/SnmpSessionListener.java => service/SnmpTransportService.java} (60%) create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java delete mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java create mode 100644 common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java create mode 100644 docker/tb-transports/snmp/conf/logback.xml create mode 100644 docker/tb-transports/snmp/conf/tb-snmp-transport.conf create mode 100644 transport/snmp/src/main/conf/logback.xml create mode 100644 transport/snmp/src/main/conf/tb-snmp-transport.conf create mode 100644 transport/snmp/src/main/resources/logback.xml create mode 100644 transport/snmp/src/main/resources/tb-snmp-transport.yml diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index 05363dfd59..f3a07eb727 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -25,7 +25,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.DefaultTbActorSystem; -import org.thingsboard.server.actors.TbActorId; import org.thingsboard.server.actors.TbActorRef; import org.thingsboard.server.actors.TbActorSystem; import org.thingsboard.server.actors.TbActorSystemSettings; @@ -33,14 +32,13 @@ import org.thingsboard.server.actors.app.AppActor; import org.thingsboard.server.actors.app.AppInitMsg; import org.thingsboard.server.actors.stats.StatsActor; import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; @Service @Slf4j diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java index d0a3984660..ccac9b4a16 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/DefaultTbApiUsageStateService.java @@ -52,7 +52,7 @@ import org.thingsboard.server.dao.usagerecord.ApiUsageStateService; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.UsageStatsKVProto; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.scheduler.SchedulerComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java index 117607ae9f..f97188597b 100644 --- a/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/apiusage/TbApiUsageStateService.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; public interface TbApiUsageStateService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 02f69ce10d..9d3e2b5a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -52,7 +52,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceM import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index 390798a3e2..bd29b6c431 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -38,7 +38,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineNotificationMsg; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbRuleEngineQueueFactory; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java index 0a965413cf..222f58fd78 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerService.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; public interface TbCoreConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java index e4a88de575..582dbd1355 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/TbRuleEngineConsumerService.java @@ -16,7 +16,7 @@ package org.thingsboard.server.service.queue; import org.springframework.context.ApplicationListener; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; public interface TbRuleEngineConsumerService extends ApplicationListener { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 02378eb557..28cce50bf5 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -34,7 +34,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index c8a4e80e0e..5c358c1b77 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -54,7 +54,7 @@ import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.util.TbCoreComponent; diff --git a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java index 932ed4720d..b1bbb9cdfa 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DeviceStateService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.service.state; import org.springframework.context.ApplicationListener; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.common.msg.queue.TbCallback; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java index ce8838aaa8..657bdf6f75 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultSubscriptionManagerService.java @@ -46,7 +46,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdate import org.thingsboard.server.gen.transport.TransportProtos.TbSubscriptionUpdateValueListProto; import org.thingsboard.server.queue.TbQueueProducer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 0220a94964..710d613b37 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -20,10 +20,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java index fd40364fe5..37850b701f 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/SubscriptionManagerService.java @@ -22,7 +22,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java index d244ef136b..f1bf6c0da0 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbLocalSubscriptionService.java @@ -15,8 +15,8 @@ */ package org.thingsboard.server.service.subscription; -import org.thingsboard.server.queue.discovery.ClusterTopologyChangeEvent; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; import org.thingsboard.server.service.telemetry.sub.TelemetrySubscriptionUpdate; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java index 168e1271fd..23abba15f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AbstractSubscriptionService.java @@ -22,35 +22,18 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationListener; import org.springframework.context.event.EventListener; -import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbApplicationEventListener; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; -import org.thingsboard.server.service.subscription.TbSubscriptionUtils; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.util.Collections; -import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java index fa44906cc3..6ed3860341 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/AlarmSubscriptionService.java @@ -17,8 +17,7 @@ package org.thingsboard.server.service.telemetry; import org.springframework.context.ApplicationListener; import org.thingsboard.rule.engine.api.RuleEngineAlarmService; -import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java index 107b97dbb3..dde71ef53f 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultAlarmSubscriptionService.java @@ -22,9 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmQuery; @@ -35,43 +33,22 @@ import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; -import org.thingsboard.server.common.data.kv.BooleanDataEntry; -import org.thingsboard.server.common.data.kv.DoubleDataEntry; -import org.thingsboard.server.common.data.kv.LongDataEntry; -import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmData; -import org.thingsboard.server.common.data.query.AlarmDataPageLink; import org.thingsboard.server.common.data.query.AlarmDataQuery; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.alarm.AlarmOperationResult; import org.thingsboard.server.dao.alarm.AlarmService; -import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; import org.thingsboard.server.service.subscription.TbSubscriptionUtils; -import org.thingsboard.server.service.telemetry.sub.AlarmSubscriptionUpdate; -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import java.util.Collection; -import java.util.Collections; -import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java index 015a58495f..0ac9e79ff1 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/TelemetrySubscriptionService.java @@ -16,8 +16,7 @@ package org.thingsboard.server.service.telemetry; import org.springframework.context.ApplicationListener; -import org.thingsboard.rule.engine.api.RuleEngineTelemetryService; -import org.thingsboard.server.queue.discovery.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; /** * Created by ashvayka on 27.03.18. diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index eea7af582c..4f58bd0f63 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; @@ -61,11 +62,15 @@ import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; @@ -84,6 +89,7 @@ import org.thingsboard.server.service.state.DeviceStateService; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -139,40 +145,41 @@ public class DefaultTransportApiService implements TransportApiService { @Override public ListenableFuture> handle(TbProtoQueueMsg tbProtoQueueMsg) { TransportApiRequestMsg transportApiRequestMsg = tbProtoQueueMsg.getValue(); + ListenableFuture result = null; + if (transportApiRequestMsg.hasValidateTokenRequestMsg()) { ValidateDeviceTokenRequestMsg msg = transportApiRequestMsg.getValidateTokenRequestMsg(); - return Futures.transform(validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg.getToken(), DeviceCredentialsType.ACCESS_TOKEN); } else if (transportApiRequestMsg.hasValidateBasicMqttCredRequestMsg()) { TransportProtos.ValidateBasicMqttCredRequestMsg msg = transportApiRequestMsg.getValidateBasicMqttCredRequestMsg(); - return Futures.transform(validateCredentials(msg), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg); } else if (transportApiRequestMsg.hasValidateX509CertRequestMsg()) { ValidateDeviceX509CertRequestMsg msg = transportApiRequestMsg.getValidateX509CertRequestMsg(); - return Futures.transform(validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg.getHash(), DeviceCredentialsType.X509_CERTIFICATE); } else if (transportApiRequestMsg.hasGetOrCreateDeviceRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getGetOrCreateDeviceRequestMsg()); } else if (transportApiRequestMsg.hasEntityProfileRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getEntityProfileRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getEntityProfileRequestMsg()); } else if (transportApiRequestMsg.hasLwM2MRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getLwM2MRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getLwM2MRequestMsg()); } else if (transportApiRequestMsg.hasValidateDeviceLwM2MCredentialsRequestMsg()) { ValidateDeviceLwM2MCredentialsRequestMsg msg = transportApiRequestMsg.getValidateDeviceLwM2MCredentialsRequestMsg(); - return Futures.transform(validateCredentials(msg.getCredentialsId(), DeviceCredentialsType.LWM2M_CREDENTIALS), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = validateCredentials(msg.getCredentialsId(), DeviceCredentialsType.LWM2M_CREDENTIALS); } else if (transportApiRequestMsg.hasProvisionDeviceRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getProvisionDeviceRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getProvisionDeviceRequestMsg()); } else if (transportApiRequestMsg.hasResourcesRequestMsg()) { - return Futures.transform(handle(transportApiRequestMsg.getResourcesRequestMsg()), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + result = handle(transportApiRequestMsg.getResourcesRequestMsg()); + } else if (transportApiRequestMsg.hasSnmpDevicesRequestMsg()) { + result = handle(transportApiRequestMsg.getSnmpDevicesRequestMsg()); + } else if (transportApiRequestMsg.hasDeviceRequestMsg()) { + result = handle(transportApiRequestMsg.getDeviceRequestMsg()); + } else if (transportApiRequestMsg.hasDeviceCredentialsRequestMsg()) { + result = handle(transportApiRequestMsg.getDeviceCredentialsRequestMsg()); } - return Futures.transform(getEmptyTransportApiResponseFuture(), - value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + + return Futures.transform(Optional.ofNullable(result).orElseGet(this::getEmptyTransportApiResponseFuture), + value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), + MoreExecutors.directExecutor()); } private ListenableFuture validateCredentials(String credentialsId, DeviceCredentialsType credentialsType) { @@ -366,6 +373,39 @@ public class DefaultTransportApiService implements TransportApiService { return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setEntityProfileResponseMsg(builder).build()); } + private ListenableFuture handle(GetDeviceRequestMsg requestMsg) { + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB())); + Device device = deviceService.findDeviceById(TenantId.SYS_TENANT_ID, deviceId); + + TransportApiResponseMsg responseMsg; + if (device != null) { + UUID deviceProfileId = device.getDeviceProfileId().getId(); + responseMsg = TransportApiResponseMsg.newBuilder() + .setDeviceResponseMsg(TransportProtos.GetDeviceResponseMsg.newBuilder() + .setDeviceProfileIdMSB(deviceProfileId.getMostSignificantBits()) + .setDeviceProfileIdLSB(deviceProfileId.getLeastSignificantBits()) + .setDeviceTransportConfiguration(ByteString.copyFrom( + dataDecodingEncodingService.encode(device.getDeviceData().getTransportConfiguration()) + ))) + .build(); + } else { + responseMsg = TransportApiResponseMsg.getDefaultInstance(); + } + + return Futures.immediateFuture(responseMsg); + } + + private ListenableFuture handle(GetDeviceCredentialsRequestMsg requestMsg) { + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB())); + DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(TenantId.SYS_TENANT_ID, deviceId); + + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder() + .setDeviceCredentialsResponseMsg(TransportProtos.GetDeviceCredentialsResponseMsg.newBuilder() + .setDeviceCredentialsData(ByteString.copyFrom(dataDecodingEncodingService.encode(deviceCredentials)))) + .build()); + } + + private ListenableFuture handle(GetResourcesRequestMsg requestMsg) { TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); TransportProtos.GetResourcesResponseMsg.Builder builder = TransportProtos.GetResourcesResponseMsg.newBuilder(); @@ -400,6 +440,20 @@ public class DefaultTransportApiService implements TransportApiService { .build(); } + // TODO: request snmp devices with pagination + private ListenableFuture handle(GetSnmpDevicesRequestMsg requestMsg) { + List result = deviceService.findDevicesIdsByDeviceProfileTransportType(DeviceTransportType.SNMP); + GetSnmpDevicesResponseMsg responseMsg = GetSnmpDevicesResponseMsg.newBuilder() + .addAllIds(result.stream() + .map(UUID::toString) + .collect(Collectors.toList())) + .build(); + + return Futures.immediateFuture(TransportApiResponseMsg.newBuilder() + .setSnmpDevicesResponseMsg(responseMsg) + .build()); + } + private ListenableFuture getDeviceInfo(DeviceId deviceId, DeviceCredentials credentials) { return Futures.transform(deviceService.findDeviceByIdAsync(TenantId.SYS_TENANT_ID, deviceId), device -> { if (device == null) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index f98c56d89a..1a893f5388 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.id.CustomerId; @@ -31,6 +32,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.dao.device.provision.ProvisionRequest; import java.util.List; +import java.util.UUID; public interface DeviceService { @@ -90,4 +92,5 @@ public interface DeviceService { Device saveDevice(ProvisionRequest provisionRequest, DeviceProfile profile); + List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java b/common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java new file mode 100644 index 0000000000..6195022aba --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TbTransportService.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +public interface TbTransportService { + String getName(); +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java index 0ae9f26cf3..49a547371f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java @@ -21,6 +21,8 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import java.io.Serializable; + @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo( use = JsonTypeInfo.Id.NAME, @@ -31,7 +33,7 @@ import org.thingsboard.server.common.data.DeviceTransportType; @JsonSubTypes.Type(value = MqttDeviceTransportConfiguration.class, name = "MQTT"), @JsonSubTypes.Type(value = Lwm2mDeviceTransportConfiguration.class, name = "LWM2M"), @JsonSubTypes.Type(value = SnmpDeviceTransportConfiguration.class, name = "SNMP")}) -public interface DeviceTransportConfiguration { +public interface DeviceTransportConfiguration extends Serializable { @JsonIgnore DeviceTransportType getType(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java index a2529e74e9..50f6c4351c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java @@ -19,14 +19,14 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.DeviceTransportType; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; import java.util.stream.Stream; @Data public class SnmpProfileTransportConfiguration implements DeviceProfileTransportConfiguration { - - private int poolPeriodMs; + private int pollPeriodMs; private int timeoutMs; private int retries; private List attributes; @@ -39,6 +39,10 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport @JsonIgnore public List getKvMappings() { - return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList()); + if (attributes != null && telemetry != null) { + return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index fc4ec33eb1..3ba360cd8e 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -19,8 +19,12 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.transport.TransportProtos; @@ -32,6 +36,7 @@ import javax.annotation.PostConstruct; import java.net.InetAddress; import java.net.UnknownHostException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -56,6 +61,8 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { @Autowired(required = false) private TbQueueRuleEngineSettings ruleEngineSettings; + @Autowired + private ApplicationContext applicationContext; private List serviceTypes; private ServiceInfo serviceInfo; @@ -102,6 +109,19 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { serviceInfo = builder.build(); } + @EventListener(ContextRefreshedEvent.class) + public void setTransports() { + serviceInfo = ServiceInfo.newBuilder(serviceInfo) + .addAllTransports(getTransportServices().stream() + .map(TbTransportService::getName) + .collect(Collectors.toSet())) + .build(); + } + + private Collection getTransportServices() { + return applicationContext.getBeansOfType(TbTransportService.class).values(); + } + @Override public ServiceInfo getServiceInfo() { return serviceInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 2da438417a..c833b314e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -15,26 +15,27 @@ */ package org.thingsboard.server.queue.discovery; -import com.google.common.hash.HashCode; import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; import com.google.common.hash.Hashing; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import org.thingsboard.server.common.msg.queue.ServiceQueue; +import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import javax.annotation.PostConstruct; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -46,7 +47,6 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ConcurrentNavigableMap; import java.util.stream.Collectors; @Service @@ -186,6 +186,8 @@ public class HashPartitionService implements PartitionService { applicationEventPublisher.publishEvent(new ClusterTopologyChangeEvent(this, changes)); } } + + applicationEventPublisher.publishEvent(new ServiceListChangedEvent(otherServices, currentService)); } @Override @@ -219,6 +221,14 @@ public class HashPartitionService implements PartitionService { } } + @Override + public int resolvePartitionIndex(UUID entityId, int partitions) { + int hash = hashFunction.newHasher() + .putLong(entityId.getMostSignificantBits()) + .putLong(entityId.getLeastSignificantBits()).hash().asInt(); + return Math.abs(hash % partitions); + } + private Map> getServiceKeyListMap(List services) { final Map> currentMap = new HashMap<>(); services.forEach(serviceInfo -> { diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java index ebb260bc57..20c59378e8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionService.java @@ -20,9 +20,11 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import java.util.List; import java.util.Set; +import java.util.UUID; /** * Once application is ready or cluster topology changes, this Service will produce {@link PartitionChangeEvent} @@ -55,4 +57,6 @@ public interface PartitionService { * @return */ TopicPartitionInfo getNotificationsTopic(ServiceType serviceType, String serviceId); + + int resolvePartitionIndex(UUID entityId, int partitions); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java index 9158d8f0c8..70484a892b 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEventListener.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.discovery; import lombok.extern.slf4j.Slf4j; import org.springframework.context.ApplicationListener; +import org.thingsboard.server.queue.discovery.event.TbApplicationEvent; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java index 5253f57248..0802f0f52c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ZkDiscoveryService.java @@ -33,12 +33,14 @@ import org.apache.zookeeper.KeeperException; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.util.Assert; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -77,7 +79,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi private volatile boolean stopped = true; - public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, PartitionService partitionService) { + public ZkDiscoveryService(TbServiceInfoProvider serviceInfoProvider, + PartitionService partitionService) { this.serviceInfoProvider = serviceInfoProvider; this.partitionService = partitionService; } @@ -126,7 +129,8 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi return; } publishCurrentServer(); - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + TransportProtos.ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); + partitionService.recalculatePartitions(currentService, getOtherServers()); } public synchronized void publishCurrentServer() { @@ -281,11 +285,11 @@ public class ZkDiscoveryService implements DiscoveryService, PathChildrenCacheLi case CHILD_ADDED: case CHILD_UPDATED: case CHILD_REMOVED: - partitionService.recalculatePartitions(serviceInfoProvider.getServiceInfo(), getOtherServers()); + TransportProtos.ServiceInfo currentService = serviceInfoProvider.getServiceInfo(); + partitionService.recalculatePartitions(currentService, getOtherServers()); break; default: break; } } - } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ClusterTopologyChangeEvent.java similarity index 91% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ClusterTopologyChangeEvent.java index 1e5b90b5fe..6b3a3ff93f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/ClusterTopologyChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ClusterTopologyChangeEvent.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.queue.discovery.event; import lombok.Getter; -import org.springframework.context.ApplicationEvent; import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import java.util.Set; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java similarity index 93% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java index 2edcd2ceca..e11e19db61 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/PartitionChangeEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/PartitionChangeEvent.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.queue.discovery.event; import lombok.Getter; -import org.springframework.context.ApplicationEvent; import org.thingsboard.server.common.msg.queue.ServiceQueueKey; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java new file mode 100644 index 0000000000..dd7d384640 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/ServiceListChangedEvent.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.discovery.event; + +import lombok.Getter; +import lombok.ToString; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; + +import java.util.List; + +@Getter +@ToString +public class ServiceListChangedEvent extends TbApplicationEvent { + private final List otherServices; + private final ServiceInfo currentService; + + public ServiceListChangedEvent(List otherServices, ServiceInfo currentService) { + super(otherServices); + this.otherServices = otherServices; + this.currentService = currentService; + } +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/TbApplicationEvent.java similarity index 95% rename from common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java rename to common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/TbApplicationEvent.java index face2d36d6..1a75b4e4e4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TbApplicationEvent.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/event/TbApplicationEvent.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.queue.discovery; +package org.thingsboard.server.queue.discovery.event; import lombok.Getter; import org.springframework.context.ApplicationEvent; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java new file mode 100644 index 0000000000..36080afaab --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/TbSnmpTransportComponent.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE, ElementType.METHOD}) +public @interface TbSnmpTransportComponent { +} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 42f6267d42..59fcdad36a 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -34,6 +34,7 @@ message ServiceInfo { int64 tenantIdMSB = 3; int64 tenantIdLSB = 4; repeated QueueInfo ruleEngineQueues = 5; + repeated string transports = 6; } /** @@ -250,6 +251,34 @@ message GetEntityProfileResponseMsg { bytes apiState = 3; } +message GetDeviceRequestMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; +} + +message GetDeviceResponseMsg { + int64 deviceProfileIdMSB = 1; + int64 deviceProfileIdLSB = 2; + bytes deviceTransportConfiguration = 3; +} + +message GetDeviceCredentialsRequestMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; +} + +message GetDeviceCredentialsResponseMsg { + bytes deviceCredentialsData = 1; +} + +message GetSnmpDevicesRequestMsg { + +} + +message GetSnmpDevicesResponseMsg { + repeated string ids = 1; +} + message EntityUpdateMsg { string entityType = 1; bytes data = 2; @@ -559,6 +588,9 @@ message TransportApiRequestMsg { ProvisionDeviceRequestMsg provisionDeviceRequestMsg = 7; ValidateDeviceLwM2MCredentialsRequestMsg validateDeviceLwM2MCredentialsRequestMsg = 8; GetResourcesRequestMsg resourcesRequestMsg = 9; + GetSnmpDevicesRequestMsg snmpDevicesRequestMsg = 10; + GetDeviceRequestMsg deviceRequestMsg = 11; + GetDeviceCredentialsRequestMsg deviceCredentialsRequestMsg = 12; } /* Response from ThingsBoard Core Service to Transport Service */ @@ -567,8 +599,11 @@ message TransportApiResponseMsg { GetOrCreateDeviceFromGatewayResponseMsg getOrCreateDeviceResponseMsg = 2; GetEntityProfileResponseMsg entityProfileResponseMsg = 3; ProvisionDeviceResponseMsg provisionDeviceResponseMsg = 4; + GetSnmpDevicesResponseMsg snmpDevicesResponseMsg = 5; LwM2MResponseMsg lwM2MResponseMsg = 6; GetResourcesResponseMsg resourcesResponseMsg = 7; + GetDeviceResponseMsg deviceResponseMsg = 8; + GetDeviceCredentialsResponseMsg deviceCredentialsResponseMsg = 9; } /* Messages that are handled by ThingsBoard Core Service */ diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index 8abc859b21..dd4a76dcc8 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -18,12 +18,12 @@ package org.thingsboard.server.transport.coap; import lombok.extern.slf4j.Slf4j; import org.eclipse.californium.core.CoapResource; import org.eclipse.californium.core.CoapServer; - import org.eclipse.californium.core.network.CoapEndpoint; import org.eclipse.californium.core.network.CoapEndpoint.Builder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.TbTransportService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -34,7 +34,7 @@ import java.net.UnknownHostException; @Service("CoapTransportService") @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')") @Slf4j -public class CoapTransportService { +public class CoapTransportService implements TbTransportService { private static final String V1 = "v1"; private static final String API = "api"; @@ -73,4 +73,9 @@ public class CoapTransportService { this.server.destroy(); log.info("CoAP transport stopped!"); } + + @Override + public String getName() { + return "COAP"; + } } diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index e222b690ef..011093149e 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -31,6 +31,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportContext; @@ -41,7 +42,6 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; @@ -53,7 +53,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import javax.servlet.http.HttpServletRequest; @@ -69,7 +68,7 @@ import java.util.function.Consumer; @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.http.enabled}'=='true')") @RequestMapping("/api/v1") @Slf4j -public class DeviceApiController { +public class DeviceApiController implements TbTransportService { @Autowired private HttpTransportContext transportContext; @@ -338,4 +337,9 @@ public class DeviceApiController { .build(), TransportServiceCallback.EMPTY); } + @Override + public String getName() { + return "HTTP"; + } + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java index 8d1aff37b5..db419eca9f 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java @@ -20,12 +20,13 @@ import org.eclipse.leshan.core.response.ReadResponse; import org.eclipse.leshan.server.registration.Registration; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import java.util.Collection; import java.util.Optional; -public interface LwM2mTransportService { +public interface LwM2mTransportService extends TbTransportService { void onRegistered(Registration registration, Collection previousObsersations); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java index 548d750a3f..bae6a0af1d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java @@ -40,6 +40,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; @@ -1118,4 +1119,9 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { namesIsWritable.addAll(new HashSet<>(keyNamesIsWritable.values())); return new ArrayList<>(namesIsWritable); } + + @Override + public String getName() { + return "LWM2M"; + } } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java index 0a0c0cbe70..3e53dec227 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportService.java @@ -28,6 +28,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.TbTransportService; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -38,7 +39,7 @@ import javax.annotation.PreDestroy; @Service("MqttTransportService") @ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.mqtt.enabled}'=='true')") @Slf4j -public class MqttTransportService { +public class MqttTransportService implements TbTransportService { @Value("${transport.mqtt.bind_address}") private String host; @@ -90,4 +91,9 @@ public class MqttTransportService { } log.info("MQTT transport stopped!"); } + + @Override + public String getName() { + return "MQTT"; + } } diff --git a/common/transport/snmp/pom.xml b/common/transport/snmp/pom.xml index c7e22ecc80..ab83ab69a7 100644 --- a/common/transport/snmp/pom.xml +++ b/common/transport/snmp/pom.xml @@ -58,9 +58,5 @@ org.snmp4j snmp4j - - org.thingsboard.common - dao-api - diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java new file mode 100644 index 0000000000..7109ef4e3f --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java @@ -0,0 +1,81 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import org.snmp4j.CommunityTarget; +import org.snmp4j.PDU; +import org.snmp4j.Snmp; +import org.snmp4j.Target; +import org.snmp4j.event.ResponseEvent; +import org.snmp4j.mp.SnmpConstants; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.VariableBinding; +import org.snmp4j.transport.DefaultUdpTransportMapping; +import org.snmp4j.transport.UdpTransportMapping; + +import java.io.IOException; + +/** + * For testing purposes. Will be removed when the time comes + */ +public class SnmpDeviceSimulator { + private final Target target; + private final OID oid = new OID(".1.3.6.1.2.1.1.1.0"); + private Snmp snmp; + + public SnmpDeviceSimulator(int port) { + String address = "udp:127.0.0.1/" + port; + + CommunityTarget target = new CommunityTarget(); + target.setCommunity(new OctetString("public")); + target.setAddress(GenericAddress.parse(address)); + target.setRetries(2); + target.setTimeout(1500); + target.setVersion(SnmpConstants.version2c); + + this.target = target; + } + + public static void main(String[] args) throws IOException { + SnmpDeviceSimulator deviceSimulator = new SnmpDeviceSimulator(161); + + deviceSimulator.start(); + String response = deviceSimulator.sendRequest(PDU.GET); + + System.out.println(response); + } + + public void start() throws IOException { + UdpTransportMapping transport = new DefaultUdpTransportMapping(); + transport.addTransportListener((sourceTransport, incomingAddress, wholeMessage, tmStateReference) -> { + System.out.println(); + }); + snmp = new Snmp(transport); + + transport.listen(); + } + + public String sendRequest(int pduType) throws IOException { + PDU pdu = new PDU(); + pdu.add(new VariableBinding(oid)); + pdu.setType(pduType); + + ResponseEvent responseEvent = snmp.send(pdu, target); + return responseEvent.getResponse().get(0).getVariable().toString(); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 66f852a209..a9fa8380fa 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -15,17 +15,19 @@ */ package org.thingsboard.server.transport.snmp; -import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; -import org.snmp4j.Snmp; import org.snmp4j.smi.OID; import org.snmp4j.smi.VariableBinding; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.stereotype.Service; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileKvMapping; import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; @@ -33,84 +35,180 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.transport.DeviceUpdatedEvent; import org.thingsboard.server.common.transport.TransportContext; -import org.thingsboard.server.dao.device.DeviceCredentialsService; -import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx; +import org.thingsboard.server.common.transport.TransportDeviceProfileCache; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; +import org.thingsboard.server.common.transport.auth.SessionInfoCreator; +import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; +import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.service.ProtoTransportEntityService; +import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService; +import org.thingsboard.server.transport.snmp.service.SnmpTransportService; +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; -@Service("SnmpTransportContext") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") +@TbSnmpTransportComponent +@Component @Slf4j +@RequiredArgsConstructor public class SnmpTransportContext extends TransportContext { - @Autowired - DeviceCredentialsService deviceCredentialsService; - - @Autowired - SnmpTransportService snmpTransportService; - - @Getter - private final Map profileTransportConfig = new ConcurrentHashMap<>(); - @Getter - private final Map> pdusPerProfile = new ConcurrentHashMap<>(); - @Getter - private final Map deviceSessions = new ConcurrentHashMap<>(); - - public Optional findAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) { - if (profileTransportConfig.containsKey(deviceProfileId)) { - return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getAttributes()); - } - return Optional.empty(); + private final SnmpTransportService snmpTransportService; + private final TransportDeviceProfileCache deviceProfileCache; + private final TransportService transportService; + private final ProtoTransportEntityService protoEntityService; + private final SnmpTransportBalancingService balancingService; + + private final Map sessions = new ConcurrentHashMap<>(); + private final Map profilesTransportConfigs = new ConcurrentHashMap<>(); + private final Map> profilesPdus = new ConcurrentHashMap<>(); + private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); + + @EventListener(ApplicationReadyEvent.class) + @Order(2) + public void initDevicesSessions() { + log.info("Initializing SNMP devices sessions"); + allSnmpDevicesIds = protoEntityService.getAllSnmpDevicesIds().stream() + .map(DeviceId::new) + .collect(Collectors.toList()); + log.trace("Found all SNMP devices ids: {}", allSnmpDevicesIds); + + List managedDevicesIds = allSnmpDevicesIds.stream() + .filter(deviceId -> balancingService.isManagedByCurrentTransport(deviceId.getId())) + .collect(Collectors.toList()); + log.info("SNMP devices managed by current SNMP transport: {}", managedDevicesIds); + + managedDevicesIds.stream() + .map(protoEntityService::getDeviceById) + .collect(Collectors.toList()) + .forEach(this::establishDeviceSession); } - public Optional findTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { - if (profileTransportConfig.containsKey(deviceProfileId)) { - return findMapping(responseOid, profileTransportConfig.get(deviceProfileId).getTelemetry()); + private void establishDeviceSession(Device device) { + if (device == null) return; + log.info("Establishing SNMP device session for device {}", device.getId()); + + DeviceProfileId deviceProfileId = device.getDeviceProfileId(); + DeviceProfile deviceProfile = deviceProfileCache.get(deviceProfileId); + + DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId()); + if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { + log.warn("[{}] Expected credentials type is {} but found {}", device.getId(), DeviceCredentialsType.ACCESS_TOKEN, credentials.getCredentialsType()); + return; } - return Optional.empty(); - } - private Optional findMapping(OID responseOid, List mappings) { - return mappings.stream() - .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) - //TODO: OID shouldn't be duplicated in the config, add backend and UI verification - .findFirst(); - } + SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + if (!deviceTransportConfiguration.isValid()) { + log.warn("SNMP device transport configuration is not valid"); + return; + } - public void initPduListPerProfile() { - profileTransportConfig.forEach(this::updatePduListPerProfile); - } + SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + profilesPdus.computeIfAbsent(deviceProfileId, id -> createPdus(profileTransportConfiguration)); - public void updatePduListPerProfile(DeviceProfileId id, SnmpProfileTransportConfiguration config) { - pdusPerProfile.put(id, createPduList(config)); + DeviceSessionContext deviceSessionContext = new DeviceSessionContext( + device, deviceProfile, + credentials.getCredentialsId(), deviceTransportConfiguration, + this, snmpTransportService + ); + registerSessionMsgListener(deviceSessionContext); + sessions.put(device.getId(), deviceSessionContext); + log.info("Established SNMP device session for device {}", device.getId()); } - public void updateDeviceSessionCtx(Device device, DeviceProfile deviceProfile, Snmp snmp) { - DeviceCredentials credentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(device.getTenantId(), device.getId()); - if (DeviceCredentialsType.ACCESS_TOKEN.equals(credentials.getCredentialsType())) { - SnmpDeviceTransportConfiguration snmpDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - if (snmpDeviceTransportConfiguration.isValid()) { - DeviceSessionCtx deviceSessionCtx = new DeviceSessionCtx(this, credentials.getCredentialsId(), snmpDeviceTransportConfiguration, snmp, device.getId(), deviceProfile); - deviceSessionCtx.createSessionInfo(ctx -> getTransportService().registerAsyncSession(deviceSessionCtx.getSessionInfo(), deviceSessionCtx)); - this.deviceSessions.put(device.getId(), deviceSessionCtx); - } - } else { + private void updateDeviceSession(DeviceSessionContext sessionContext, Device device, DeviceProfile deviceProfile) { + log.info("Updating SNMP device session for device {}", device.getId()); + DeviceProfileId deviceProfileId = deviceProfile.getId(); + + DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId()); + if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { log.warn("[{}] Expected credentials type is {} but found {}", device.getId(), DeviceCredentialsType.ACCESS_TOKEN, credentials.getCredentialsType()); + destroyDeviceSession(sessionContext); + return; + } + + SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + sessionContext.setProfileTransportConfiguration(profileTransportConfiguration); + sessionContext.setDeviceTransportConfiguration(deviceTransportConfiguration); + if (!deviceTransportConfiguration.isValid()) { + log.warn("SNMP device transport configuration is not valid"); + destroyDeviceSession(sessionContext); + return; + } + + if (!profileTransportConfiguration.equals(profilesTransportConfigs.get(deviceProfileId))) { + profilesPdus.put(deviceProfileId, createPdus(profileTransportConfiguration)); + profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration); + sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); + } else if (!deviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { + sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); + } else { + log.trace("Configuration of the device {} was not updated", device); } } - public ExecutorService getSnmpCallbackExecutor() { - return snmpTransportService.getSnmpCallbackExecutor(); + private void destroyDeviceSession(DeviceSessionContext sessionContext) { + if (sessionContext == null) return; + log.info("Destroying SNMP device session for device {}", sessionContext.getDevice().getId()); + sessionContext.close(); + transportService.deregisterSession(sessionContext.getSessionInfo()); + sessions.remove(sessionContext.getDeviceId()); + log.trace("Deregistered and removed session"); + + DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); + if (sessions.values().stream() + .map(DeviceAwareSessionContext::getDeviceProfile) + .noneMatch(deviceProfile -> deviceProfile.getId().equals(deviceProfileId))) { + log.trace("Removed values for device profile {} from configs and pdus caches", deviceProfileId); + profilesTransportConfigs.remove(deviceProfileId); + profilesPdus.remove(deviceProfileId); + } } - private List createPduList(SnmpProfileTransportConfiguration deviceProfileConfig) { + private void registerSessionMsgListener(DeviceSessionContext deviceSessionContext) { + transportService.process(DeviceTransportType.SNMP, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceSessionContext.getToken()).build(), + new TransportServiceCallback() { + @Override + public void onSuccess(ValidateDeviceCredentialsResponse msg) { + if (msg.hasDeviceInfo()) { + SessionInfoProto sessionInfo = SessionInfoCreator.create( + msg, SnmpTransportContext.this, UUID.randomUUID() + ); + + transportService.registerAsyncSession(sessionInfo, deviceSessionContext); + deviceSessionContext.setSessionInfo(sessionInfo); + deviceSessionContext.setDeviceInfo(msg.getDeviceInfo()); + } else { + log.warn("[{}] Failed to process device auth", deviceSessionContext.getDeviceId()); + } + } + + @Override + public void onError(Throwable e) { + log.warn("[{}] Failed to process device auth: {}", deviceSessionContext.getDeviceId(), e); + } + }); + } + + private List createPdus(SnmpProfileTransportConfiguration deviceProfileConfig) { Map> varBindingPerMethod = new HashMap<>(); deviceProfileConfig.getKvMappings().forEach(mapping -> varBindingPerMethod @@ -127,7 +225,106 @@ public class SnmpTransportContext extends TransportContext { .collect(Collectors.toList()); } - //TODO: Extract SNMP methods to enum + @EventListener(DeviceUpdatedEvent.class) + public void onDeviceUpdatedOrCreated(DeviceUpdatedEvent deviceUpdatedEvent) { + Device device = deviceUpdatedEvent.getDevice(); + log.trace("Got creating or updating device event for device {}", device); + DeviceTransportType transportType = Optional.ofNullable(device.getDeviceData().getTransportConfiguration()) + .map(DeviceTransportConfiguration::getType) + .orElse(null); + if (!allSnmpDevicesIds.contains(device.getId())) { + if (transportType != DeviceTransportType.SNMP) { + return; + } + allSnmpDevicesIds.add(device.getId()); + if (balancingService.isManagedByCurrentTransport(device.getId().getId())) { + establishDeviceSession(device); + } + } else { + if (balancingService.isManagedByCurrentTransport(device.getId().getId())) { + DeviceSessionContext sessionContext = sessions.get(device.getId()); + if (transportType == DeviceTransportType.SNMP) { + if (sessionContext != null) { + updateDeviceSession(sessionContext, device, deviceProfileCache.get(device.getDeviceProfileId())); + } else { + establishDeviceSession(device); + } + } else { + log.trace("Transport type was changed to {}", transportType); + destroyDeviceSession(sessionContext); + } + } + } + } + + public void onDeviceDeleted(DeviceSessionContext sessionContext) { + destroyDeviceSession(sessionContext); + } + + public void onDeviceProfileUpdated(DeviceProfile deviceProfile, DeviceSessionContext sessionContext) { + updateDeviceSession(sessionContext, sessionContext.getDevice(), deviceProfile); + } + + public void onSnmpTransportListChanged() { + log.trace("SNMP transport list changed. Updating sessions"); + List deleted = new LinkedList<>(); + for (DeviceId deviceId : allSnmpDevicesIds) { + if (balancingService.isManagedByCurrentTransport(deviceId.getId())) { + if (!sessions.containsKey(deviceId)) { + Device device = protoEntityService.getDeviceById(deviceId); + if (device != null) { + log.info("SNMP device {} is now managed by current transport node", deviceId); + establishDeviceSession(device); + } else { + deleted.add(deviceId); + } + } + } else { + Optional.ofNullable(sessions.get(deviceId)) + .ifPresent(sessionContext -> { + log.info("SNMP session for device {} is not managed by current transport node anymore", deviceId); + destroyDeviceSession(sessionContext); + }); + } + } + log.trace("Removing deleted SNMP devices: {}", deleted); + allSnmpDevicesIds.removeAll(deleted); + } + + + public Collection getSessions() { + return sessions.values(); + } + + public Map> getProfilesPdus() { + return profilesPdus; + } + + public Optional getAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) { + if (profilesTransportConfigs.containsKey(deviceProfileId)) { + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributes()); + } + return Optional.empty(); + } + + public Optional getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { + if (profilesTransportConfigs.containsKey(deviceProfileId)) { + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetry()); + } + return Optional.empty(); + } + + private Optional getMapping(OID responseOid, List mappings) { + return mappings.stream() + .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) + //TODO: OID shouldn't be duplicated in the config, add backend and UI verification + .findFirst(); + } + + public ExecutorService getSnmpCallbackExecutor() { + return snmpTransportService.getSnmpCallbackExecutor(); + } + private int getSnmpMethod(String configMethod) { switch (configMethod) { case "get": diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java deleted file mode 100644 index 5d8e4d84b4..0000000000 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportService.java +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.snmp; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.snmp4j.Snmp; -import org.snmp4j.transport.DefaultUdpTransportMapping; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.DeviceInfo; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageDataIterable; -import org.thingsboard.server.dao.device.DeviceProfileService; -import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.transport.snmp.session.DeviceSessionCtx; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; -import java.io.IOException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -@Service("SnmpTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.snmp.enabled}'=='true')") -@Slf4j -public class SnmpTransportService { - - private static final int ENTITY_PACK_LIMIT = 1024; - - @Autowired - private SnmpTransportContext snmpTransportContext; - - @Autowired - DeviceProfileService deviceProfileService; - - @Autowired - TenantService tenantService; - - @Autowired - DeviceService deviceService; - - @Getter - private ExecutorService snmpCallbackExecutor; - private Snmp snmp; - private ScheduledExecutorService pollingExecutor; - - @PostConstruct - public void init() { - log.info("Starting SNMP transport..."); - pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); - //TODO: Set parallelism value in the config - snmpCallbackExecutor = Executors.newWorkStealingPool(20); - initializeSnmp(); - log.info("SNMP transport started!"); - } - - @PreDestroy - public void shutdown() { - log.info("Stopping SNMP transport!"); - if (pollingExecutor != null) { - pollingExecutor.shutdownNow(); - } - if (snmpCallbackExecutor != null) { - snmpCallbackExecutor.shutdownNow(); - } - if (snmp != null) { - try { - snmp.close(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - log.info("SNMP transport stopped!"); - } - - @EventListener(ApplicationReadyEvent.class) - @Order(value = 2) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting SNMP polling."); - initSessionCtxList(); - startPolling(); - } - - private void initializeSnmp() { - try { - this.snmp = new Snmp(new DefaultUdpTransportMapping()); - this.snmp.listen(); - } catch (IOException e) { - //TODO: what should be done if transport wasn't initialized? - log.error(e.getMessage(), e); - } - } - - private void initSessionCtxList() { - //TODO: This approach works for monolith, in cluster the same data will be fetched by each node. - for (Tenant tenant : new PageDataIterable<>(tenantService::findTenants, ENTITY_PACK_LIMIT)) { - TenantId tenantId = tenant.getTenantId(); - for (DeviceProfile deviceProfile : new PageDataIterable<>(pageLink -> deviceProfileService.findDeviceProfiles(tenantId, pageLink), ENTITY_PACK_LIMIT)) { - if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) { - snmpTransportContext.getProfileTransportConfig().put(deviceProfile.getId(), - (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration()); - initDeviceSessions(deviceProfile); - } - } - } - snmpTransportContext.initPduListPerProfile(); - } - - private void initDeviceSessions(DeviceProfile deviceProfile) { - for (DeviceInfo deviceInfo : new PageDataIterable<>(pageLink -> deviceService.findDeviceInfosByTenantIdAndDeviceProfileId(deviceProfile.getTenantId(), deviceProfile.getId(), pageLink), ENTITY_PACK_LIMIT)) { - snmpTransportContext.updateDeviceSessionCtx(deviceInfo, deviceProfile, snmp); - } - } - - private void startPolling() { - //TODO: Get poll period from configuration; - int poolPeriodSeconds = 1; - pollingExecutor.scheduleAtFixedRate(() -> snmpTransportContext.getDeviceSessions().values().forEach(DeviceSessionCtx::executeSnmpRequest), - 0, poolPeriodSeconds, TimeUnit.SECONDS); - } -} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java new file mode 100644 index 0000000000..90b82abf96 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/ServiceListChangedEventListener.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.event; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService; + +@TbSnmpTransportComponent +@Component +@RequiredArgsConstructor +public class ServiceListChangedEventListener extends TbApplicationEventListener { + private final SnmpTransportBalancingService snmpTransportBalancingService; + + @Override + protected void onTbApplicationEvent(ServiceListChangedEvent event) { + snmpTransportBalancingService.onServiceListChanged(event); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java new file mode 100644 index 0000000000..f7a4259b91 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEvent.java @@ -0,0 +1,24 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.event; + +import org.thingsboard.server.queue.discovery.event.TbApplicationEvent; + +public class SnmpTransportListChangedEvent extends TbApplicationEvent { + public SnmpTransportListChangedEvent() { + super(new Object()); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java new file mode 100644 index 0000000000..1387461037 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/event/SnmpTransportListChangedEventListener.java @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.event; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.discovery.TbApplicationEventListener; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; + +@TbSnmpTransportComponent +@Component +@RequiredArgsConstructor +public class SnmpTransportListChangedEventListener extends TbApplicationEventListener { + private final SnmpTransportContext snmpTransportContext; + + @Override + protected void onTbApplicationEvent(SnmpTransportListChangedEvent event) { + snmpTransportContext.onSnmpTransportListChanged(); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java new file mode 100644 index 0000000000..1f639daa2a --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; + +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +@TbSnmpTransportComponent +@Service +@RequiredArgsConstructor +public class ProtoTransportEntityService { + private final TransportService transportService; + private final DataDecodingEncodingService dataDecodingEncodingService; + + public Device getDeviceById(DeviceId id) { + TransportProtos.GetDeviceResponseMsg deviceProto = transportService.getDevice(TransportProtos.GetDeviceRequestMsg.newBuilder() + .setDeviceIdMSB(id.getId().getMostSignificantBits()) + .setDeviceIdLSB(id.getId().getLeastSignificantBits()) + .build()); + + if (deviceProto == null) { + return null; + } + + DeviceProfileId deviceProfileId = new DeviceProfileId(new UUID( + deviceProto.getDeviceProfileIdMSB(), deviceProto.getDeviceProfileIdLSB()) + ); + + Device device = new Device(); + device.setId(id); + device.setDeviceProfileId(deviceProfileId); + + DeviceTransportConfiguration deviceTransportConfiguration = (DeviceTransportConfiguration) dataDecodingEncodingService.decode( + deviceProto.getDeviceTransportConfiguration().toByteArray() + ).orElseThrow(() -> new IllegalStateException("Can't find device transport configuration")); + + DeviceData deviceData = new DeviceData(); + deviceData.setTransportConfiguration(deviceTransportConfiguration); + device.setDeviceData(deviceData); + + return device; + } + + public DeviceCredentials getDeviceCredentialsByDeviceId(DeviceId deviceId) { + TransportProtos.GetDeviceCredentialsResponseMsg deviceCredentialsResponse = transportService.getDeviceCredentials( + TransportProtos.GetDeviceCredentialsRequestMsg.newBuilder() + .setDeviceIdMSB(deviceId.getId().getMostSignificantBits()) + .setDeviceIdLSB(deviceId.getId().getLeastSignificantBits()) + .build() + ); + + return (DeviceCredentials) dataDecodingEncodingService.decode(deviceCredentialsResponse.getDeviceCredentialsData().toByteArray()) + .orElseThrow(() -> new IllegalArgumentException("Device credentials not found")); + } + + public List getAllSnmpDevicesIds() { + TransportProtos.GetSnmpDevicesResponseMsg devicesIdsResponse = transportService.getSnmpDevicesIds( + TransportProtos.GetSnmpDevicesRequestMsg.getDefaultInstance() + ); + + return devicesIdsResponse.getIdsList().stream() + .map(UUID::fromString) + .collect(Collectors.toList()); + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java new file mode 100644 index 0000000000..3564ab319e --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportBalancingService.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; +import org.thingsboard.server.queue.discovery.PartitionService; +import org.thingsboard.server.queue.discovery.event.ServiceListChangedEvent; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.event.SnmpTransportListChangedEvent; + +import java.util.Comparator; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@TbSnmpTransportComponent +@Service +@RequiredArgsConstructor +@Slf4j +public class SnmpTransportBalancingService { + private final PartitionService partitionService; + private final ApplicationEventPublisher eventPublisher; + private final SnmpTransportService snmpTransportService; + + private int snmpTransportsCount = 1; + private Integer currentTransportPartitionIndex = 0; + + public void onServiceListChanged(ServiceListChangedEvent event) { + log.trace("Got service list changed event: {}", event); + recalculatePartitions(event.getOtherServices(), event.getCurrentService()); + } + + public boolean isManagedByCurrentTransport(UUID entityId) { + boolean isManaged = resolvePartitionIndexForEntity(entityId) == currentTransportPartitionIndex; + if (!isManaged) { + log.trace("Entity {} is not managed by current SNMP transport node", entityId); + } + return isManaged; + } + + private int resolvePartitionIndexForEntity(UUID entityId) { + return partitionService.resolvePartitionIndex(entityId, snmpTransportsCount); + } + + private void recalculatePartitions(List otherServices, ServiceInfo currentService) { + log.info("Recalculating partitions for SNMP transports"); + List snmpTransports = Stream.concat(otherServices.stream(), Stream.of(currentService)) + .filter(service -> service.getTransportsList().contains(snmpTransportService.getName())) + .sorted(Comparator.comparing(ServiceInfo::getServiceId)) + .collect(Collectors.toList()); + log.trace("Found SNMP transports: {}", snmpTransports); + + int previousCurrentTransportPartitionIndex = currentTransportPartitionIndex; + int previousSnmpTransportsCount = snmpTransportsCount; + + if (!snmpTransports.isEmpty()) { + for (int i = 0; i < snmpTransports.size(); i++) { + if (snmpTransports.get(i).equals(currentService)) { + currentTransportPartitionIndex = i; + break; + } + } + snmpTransportsCount = snmpTransports.size(); + } + + if (snmpTransportsCount != previousSnmpTransportsCount || currentTransportPartitionIndex != previousCurrentTransportPartitionIndex) { + log.info("SNMP transports partitions have changed: transports count = {}, current transport partition index = {}", snmpTransportsCount, currentTransportPartitionIndex); + eventPublisher.publishEvent(new SnmpTransportListChangedEvent()); + } else { + log.info("SNMP transports partitions have not changed"); + } + } + +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java similarity index 60% rename from common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java rename to common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 35aa1a9347..22e60c98ee 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/SnmpSessionListener.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.snmp.session; +package org.thingsboard.server.transport.snmp.service; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -25,9 +25,16 @@ import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.Snmp; import org.snmp4j.event.ResponseEvent; -import org.snmp4j.event.ResponseListener; import org.snmp4j.smi.VariableBinding; +import org.snmp4j.transport.DefaultUdpTransportMapping; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.annotation.Lazy; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.transport.TransportContext; @@ -39,27 +46,116 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.snmp.SnmpTransportContext; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; +import javax.annotation.PreDestroy; +import java.io.IOException; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +@TbSnmpTransportComponent +@Service @Slf4j -@AllArgsConstructor -public class SnmpSessionListener implements ResponseListener { - - @Getter +public class SnmpTransportService implements TbTransportService { private final SnmpTransportContext snmpTransportContext; @Getter - private final String token; + private ExecutorService snmpCallbackExecutor; + @Getter + private Snmp snmp; + private ScheduledExecutorService pollingExecutor; - @Override - public void onResponse(ResponseEvent event) { - ((Snmp) event.getSource()).cancel(event.getRequest(), this); - snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(event)); + public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext) { + this.snmpTransportContext = snmpTransportContext; } - private void processSnmpResponse(ResponseEvent event) { + // @PostConstruct + private void init() { + log.info("Starting SNMP transport..."); + pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); + //TODO: Set parallelism value in the config + snmpCallbackExecutor = Executors.newWorkStealingPool(20); + + initializeSnmp(); + + log.info("SNMP transport started!"); + } + + @PreDestroy + public void shutdown() { + log.info("Stopping SNMP transport!"); + if (pollingExecutor != null) { + pollingExecutor.shutdownNow(); + } + if (snmpCallbackExecutor != null) { + snmpCallbackExecutor.shutdownNow(); + } + if (snmp != null) { + try { + snmp.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + log.info("SNMP transport stopped!"); + } + + @EventListener(ApplicationReadyEvent.class) + @Order(value = 10) + public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { + log.info("Received application ready event. Starting SNMP polling."); +// startPolling(); + } + + private void initializeSnmp() { + try { + this.snmp = new Snmp(new DefaultUdpTransportMapping()); + this.snmp.listen(); + } catch (IOException e) { + //TODO: what should be done if transport wasn't initialized? + log.error(e.getMessage(), e); + } + } + + private void startPolling() { + //TODO: Get poll period from configuration; + int pollPeriodSeconds = 1; + + pollingExecutor.scheduleWithFixedDelay(() -> { + snmpTransportContext.getSessions().forEach(this::executeSnmpRequest); + }, 0, pollPeriodSeconds, TimeUnit.SECONDS); + } + + private void executeSnmpRequest(DeviceSessionContext sessionContext) { + long timeNow = System.currentTimeMillis(); + long nextRequestExecutionTime = sessionContext.getPreviousRequestExecutedAt() + sessionContext.getProfileTransportConfiguration().getPollPeriodMs(); + + if (nextRequestExecutionTime < timeNow) { + sessionContext.setPreviousRequestExecutedAt(timeNow); + + DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); + snmpTransportContext.getProfilesPdus().get(deviceProfileId).forEach(pdu -> { + try { + log.debug("[{}] Sending SNMP message...", pdu.getRequestID()); + snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + }); + } + } + + public void onNewDeviceResponse(ResponseEvent responseEvent, DeviceSessionContext sessionContext) { + ((Snmp) responseEvent.getSource()).cancel(responseEvent.getRequest(), sessionContext); + snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(responseEvent, sessionContext)); + } + + private void processSnmpResponse(ResponseEvent event, DeviceSessionContext sessionContext) { PDU response = event.getResponse(); if (event.getError() != null) { log.warn("Response error: {}", event.getError().getMessage(), event.getError()); @@ -72,8 +168,8 @@ public class SnmpSessionListener implements ResponseListener { TransportService transportService = snmpTransportContext.getTransportService(); for (int i = 0; i < response.size(); i++) { VariableBinding vb = response.get(i); - snmpTransportContext.findAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(), + snmpTransportContext.getAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { try { transportService.process(sessionInfo, @@ -84,8 +180,8 @@ public class SnmpSessionListener implements ResponseListener { log.warn("Failed to process SNMP response: {}", e.getMessage(), e); } }))); - snmpTransportContext.findTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(token).build(), + snmpTransportContext.getTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, + TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { try { transportService.process(sessionInfo, @@ -103,6 +199,14 @@ public class SnmpSessionListener implements ResponseListener { } } + private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { + snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() + .setAttributeSubscription(false) + .setRpcSubscription(false) + .setLastActivityTime(System.currentTimeMillis()) + .build(), TransportServiceCallback.EMPTY); + } + private TransportProtos.PostAttributeMsg convertToPostAttributes(String keyName, DataType dataType, String payload) throws AdaptorException { try { return JsonConverter.convertToAttributesProto(getKvJson(keyName, dataType, payload)); @@ -143,14 +247,6 @@ public class SnmpSessionListener implements ResponseListener { return new JsonParser().parse(result.toString()); } - private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(false) - .setRpcSubscription(false) - .setLastActivityTime(System.currentTimeMillis()) - .build(), TransportServiceCallback.EMPTY); - } - @AllArgsConstructor private static class DeviceAuthCallback implements TransportServiceCallback { private final TransportContext transportContext; @@ -170,4 +266,9 @@ public class SnmpSessionListener implements ResponseListener { log.warn("Failed to process device auth", e); } } + + @Override + public String getName() { + return "SNMP"; + } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java new file mode 100644 index 0000000000..ed87bbc726 --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -0,0 +1,166 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.session; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.snmp4j.CommunityTarget; +import org.snmp4j.Target; +import org.snmp4j.event.ResponseEvent; +import org.snmp4j.event.ResponseListener; +import org.snmp4j.mp.SnmpConstants; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.OctetString; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; +import org.thingsboard.server.gen.transport.TransportProtos; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; +import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; +import org.thingsboard.server.transport.snmp.service.SnmpTransportService; + +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class DeviceSessionContext extends DeviceAwareSessionContext implements SessionMsgListener, ResponseListener { + @Getter + private Target target; + private final String token; + @Getter + @Setter + private SnmpProfileTransportConfiguration profileTransportConfiguration; + @Getter + @Setter + private SnmpDeviceTransportConfiguration deviceTransportConfiguration; + @Getter + private final Device device; + + private final SnmpTransportContext snmpTransportContext; + private final SnmpTransportService snmpTransportService; + + @Getter + @Setter + private long previousRequestExecutedAt = 0; + private final AtomicInteger msgIdSeq = new AtomicInteger(0); + private boolean isActive = true; + + public DeviceSessionContext(Device device, DeviceProfile deviceProfile, + String token, SnmpDeviceTransportConfiguration deviceTransportConfiguration, + SnmpTransportContext snmpTransportContext, SnmpTransportService snmpTransportService) { + super(UUID.randomUUID()); + super.setDeviceId(device.getId()); + super.setDeviceProfile(deviceProfile); + this.device = device; + + this.token = token; + this.snmpTransportContext = snmpTransportContext; + this.snmpTransportService = snmpTransportService; + + this.profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + this.deviceTransportConfiguration = deviceTransportConfiguration; + + initTarget(this.profileTransportConfiguration, this.deviceTransportConfiguration); + } + + @Override + public void onDeviceProfileUpdate(TransportProtos.SessionInfoProto newSessionInfo, DeviceProfile deviceProfile) { + super.onDeviceProfileUpdate(newSessionInfo, deviceProfile); + if (isActive) { + snmpTransportContext.onDeviceProfileUpdated(deviceProfile, this); + } + } + + @Override + public void onDeviceDeleted(DeviceId deviceId) { + snmpTransportContext.onDeviceDeleted(this); + } + + @Override + public void onResponse(ResponseEvent event) { + if (isActive) { + snmpTransportService.onNewDeviceResponse(event, this); + } + } + + public void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + log.trace("Initializing target for SNMP session of device {}", device); + CommunityTarget communityTarget = new CommunityTarget(); + communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); + communityTarget.setVersion(getSnmpVersion(deviceTransportConfig.getProtocolVersion())); + communityTarget.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); + communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); + communityTarget.setRetries(profileTransportConfig.getRetries()); + this.target = communityTarget; + log.info("SNMP target initialized: {}", this.target); + } + + public void close() { + isActive = false; + } + + public String getToken() { + return token; + } + + //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer) + private int getSnmpVersion(String configSnmpVersion) { + switch (configSnmpVersion) { + case ("v1"): + return SnmpConstants.version1; + case ("v2c"): + return SnmpConstants.version2c; + case ("v3"): + return SnmpConstants.version3; + default: + return -1; + } + } + + @Override + public int nextMsgId() { + return msgIdSeq.incrementAndGet(); + } + + @Override + public void onGetAttributesResponse(GetAttributeResponseMsg getAttributesResponse) { + } + + @Override + public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { + } + + @Override + public void onRemoteSessionCloseCommand(SessionCloseNotificationProto sessionCloseNotification) { + } + + @Override + public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) { + } + + @Override + public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) { + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java deleted file mode 100644 index 424b964387..0000000000 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionCtx.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.snmp.session; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.snmp4j.CommunityTarget; -import org.snmp4j.Snmp; -import org.snmp4j.Target; -import org.snmp4j.mp.SnmpConstants; -import org.snmp4j.smi.GenericAddress; -import org.snmp4j.smi.OctetString; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.transport.SessionMsgListener; -import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.auth.SessionInfoCreator; -import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; -import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; -import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.transport.snmp.SnmpTransportContext; - -import java.io.IOException; -import java.util.Optional; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; - -@Slf4j -public class DeviceSessionCtx extends DeviceAwareSessionContext implements SessionMsgListener { - private final AtomicInteger msgIdSeq = new AtomicInteger(0); - - @Getter - @Setter - private SnmpDeviceTransportConfiguration deviceTransportConfig; - @Getter - @Setter - private SnmpSessionListener snmpSessionListener; - @Getter - @Setter - private Target target; - @Getter - @Setter - private volatile TransportProtos.SessionInfoProto sessionInfo; - - private Snmp snmp; - private SnmpProfileTransportConfiguration snmpProfileTransportConfiguration; - private long previousRequestExecutedAt = 0; - - public DeviceSessionCtx(SnmpTransportContext transportContext, String token, SnmpDeviceTransportConfiguration deviceTransportConfig, - Snmp snmp, DeviceId deviceId, DeviceProfile deviceProfile) { - super(UUID.randomUUID()); - this.snmpSessionListener = new SnmpSessionListener(transportContext, token); - super.setDeviceId(deviceId); - super.setDeviceProfile(deviceProfile); - //TODO: What should be done if snmp null? - if (snmp != null) { - this.snmp = snmp; - } - this.snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - initTarget(this.snmpProfileTransportConfiguration, deviceTransportConfig); - } - - @Override - public int nextMsgId() { - return msgIdSeq.incrementAndGet(); - } - - @Override - public void onGetAttributesResponse(GetAttributeResponseMsg getAttributesResponse) { - } - - @Override - public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { - } - - @Override - public void onRemoteSessionCloseCommand(SessionCloseNotificationProto sessionCloseNotification) { - } - - @Override - public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) { - } - - @Override - public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) { - } - - @Override - public void onDeviceProfileUpdate(TransportProtos.SessionInfoProto newSessionInfo, DeviceProfile deviceProfile) { - super.onDeviceProfileUpdate(sessionInfo, deviceProfile); - if (DeviceTransportType.SNMP.equals(deviceProfile.getTransportType())) { - snmpProfileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - snmpSessionListener.getSnmpTransportContext().getProfileTransportConfig().put( - deviceProfile.getId(), - snmpProfileTransportConfiguration); - snmpSessionListener.getSnmpTransportContext().updatePduListPerProfile(deviceProfile.getId(), snmpProfileTransportConfiguration); - } else { - //TODO: should the context be removed from the map? - } - } - - @Override - public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { - super.onDeviceUpdate(sessionInfo, device, deviceProfileOpt); - if (super.getDeviceProfile() != null && DeviceTransportType.SNMP.equals(super.getDeviceProfile().getTransportType())) { - snmpSessionListener.getSnmpTransportContext().updateDeviceSessionCtx(device, deviceProfile, null); - SnmpProfileTransportConfiguration profileTransportConfig = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - SnmpDeviceTransportConfiguration deviceTransportConfig = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - initTarget(profileTransportConfig, deviceTransportConfig); - } else { - //TODO: should the context be removed from the map? - } - } - - public void createSessionInfo(Consumer registerSession) { - getSnmpSessionListener().getSnmpTransportContext().getTransportService().process(DeviceTransportType.SNMP, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(getSnmpSessionListener().getToken()).build(), - new TransportServiceCallback() { - @Override - public void onSuccess(ValidateDeviceCredentialsResponse msg) { - if (msg.hasDeviceInfo()) { - sessionInfo = SessionInfoCreator.create(msg, getSnmpSessionListener().getSnmpTransportContext(), UUID.randomUUID()); - registerSession.accept(sessionInfo); - setDeviceInfo(msg.getDeviceInfo()); - } else { - log.warn("[{}] Failed to process device auth", getDeviceId()); - } - } - - @Override - public void onError(Throwable e) { - log.warn("[{}] Failed to process device auth", getDeviceId(), e); - } - }); - } - - public void executeSnmpRequest() { - long timeNow = System.currentTimeMillis(); - long nextRequestExecutionTime = previousRequestExecutedAt + snmpProfileTransportConfiguration.getPoolPeriodMs(); - if (nextRequestExecutionTime < timeNow) { - previousRequestExecutedAt = timeNow; - - snmpSessionListener.getSnmpTransportContext().getPdusPerProfile().get(deviceProfile.getId()).forEach(pdu -> { - try { - log.debug("[{}] Sending SNMP message...", pdu.getRequestID()); - snmp.send(pdu, - target, - deviceProfile.getId(), - snmpSessionListener); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - }); - } - } - - private void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { - this.deviceTransportConfig = deviceTransportConfig; - CommunityTarget communityTarget = new CommunityTarget(); - communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + this.deviceTransportConfig.getAddress() + "/" + this.deviceTransportConfig.getPort())); - communityTarget.setVersion(getSnmpVersion(this.deviceTransportConfig.getProtocolVersion())); - communityTarget.setCommunity(new OctetString(this.deviceTransportConfig.getCommunity())); - communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); - communityTarget.setRetries(profileTransportConfig.getRetries()); - this.target = communityTarget; - log.info("SNMP target initialized: {}", this.target); - } - - //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer) - private int getSnmpVersion(String configSnmpVersion) { - switch (configSnmpVersion) { - case ("v1"): - return SnmpConstants.version1; - case ("v2c"): - return SnmpConstants.version2c; - case ("v3"): - return SnmpConstants.version3; - default: - return -1; - } - } -} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java new file mode 100644 index 0000000000..ad4819fbb1 --- /dev/null +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/DeviceUpdatedEvent.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.transport; + +import lombok.Getter; +import org.thingsboard.server.common.data.Device; + +@Getter +public class DeviceUpdatedEvent { + private final Device device; + + public DeviceUpdatedEvent(Device device) { + this.device = device; + } +} diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java index 8209fe2531..9aff642708 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/SessionMsgListener.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.transport; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; @@ -49,4 +50,7 @@ public interface SessionMsgListener { default void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { } + + default void onDeviceDeleted(DeviceId deviceId) { + } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index 75f20cc825..eabcb1e8e6 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -21,6 +21,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.scheduler.SchedulerComponent; import org.thingsboard.server.queue.util.TbTransportComponent; diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index ab728af00d..24469dc599 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -22,11 +22,17 @@ import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsRes import org.thingsboard.server.common.transport.service.SessionMetaData; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceCredentialsResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourcesResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.LwM2MRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.LwM2MResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; @@ -55,6 +61,12 @@ public interface TransportService { GetResourcesResponseMsg getResources(GetResourcesRequestMsg msg); + GetSnmpDevicesResponseMsg getSnmpDevicesIds(GetSnmpDevicesRequestMsg requestMsg); + + GetDeviceResponseMsg getDevice(GetDeviceRequestMsg requestMsg); + + GetDeviceCredentialsResponseMsg getDeviceCredentials(GetDeviceCredentialsRequestMsg requestMsg); + void process(DeviceTransportType transportType, ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java index 5c36e9b3e0..ad289a7e16 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportDeviceProfileCache.java @@ -112,8 +112,8 @@ public class DefaultTransportDeviceProfileCache implements TransportDeviceProfil profile = profileOpt.get(); this.put(profile); } else { - log.warn("[{}] Can't device profile: {}", id, entityProfileMsg.getData()); - throw new RuntimeException("Can't device profile!"); + log.warn("[{}] Can't find device profile: {}", id, entityProfileMsg.getData()); + throw new RuntimeException("Can't find device profile!"); } } finally { deviceProfileFetchLock.unlock(); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index b273ee48cf..903de94020 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -23,6 +23,7 @@ import com.google.gson.JsonObject; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.ApiUsageRecordKey; @@ -47,6 +48,7 @@ import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.stats.MessagesStats; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; +import org.thingsboard.server.common.transport.DeviceUpdatedEvent; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; import org.thingsboard.server.common.transport.TransportService; @@ -61,7 +63,6 @@ import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToRuleEngineMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToTransportMsg; @@ -131,6 +132,7 @@ public class DefaultTransportService implements TransportService { private final TransportRateLimitService rateLimitService; private final DataDecodingEncodingService dataDecodingEncodingService; private final SchedulerComponent scheduler; + private final ApplicationEventPublisher eventPublisher; protected TbQueueRequestTemplate, TbProtoQueueMsg> transportApiRequestTemplate; protected TbQueueProducer> ruleEngineMsgProducer; @@ -157,7 +159,8 @@ public class DefaultTransportService implements TransportService { TransportDeviceProfileCache deviceProfileCache, TransportTenantProfileCache tenantProfileCache, TbApiUsageClient apiUsageClient, TransportRateLimitService rateLimitService, - DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler) { + DataDecodingEncodingService dataDecodingEncodingService, SchedulerComponent scheduler, + ApplicationEventPublisher eventPublisher) { this.serviceInfoProvider = serviceInfoProvider; this.queueProvider = queueProvider; this.producerProvider = producerProvider; @@ -169,6 +172,7 @@ public class DefaultTransportService implements TransportService { this.rateLimitService = rateLimitService; this.dataDecodingEncodingService = dataDecodingEncodingService; this.scheduler = scheduler; + this.eventPublisher = eventPublisher; } @PostConstruct @@ -266,6 +270,58 @@ public class DefaultTransportService implements TransportService { } } + @Override + public TransportProtos.GetSnmpDevicesResponseMsg getSnmpDevicesIds(TransportProtos.GetSnmpDevicesRequestMsg requestMsg) { + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>( + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder() + .setSnmpDevicesRequestMsg(requestMsg) + .build() + ); + + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getSnmpDevicesResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @Override + public TransportProtos.GetDeviceResponseMsg getDevice(TransportProtos.GetDeviceRequestMsg requestMsg) { + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>( + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder() + .setDeviceRequestMsg(requestMsg) + .build() + ); + + try { + TransportApiResponseMsg response = transportApiRequestTemplate.send(protoMsg).get().getValue(); + if (response.hasDeviceResponseMsg()) { + return response.getDeviceResponseMsg(); + } else { + return null; + } + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @Override + public TransportProtos.GetDeviceCredentialsResponseMsg getDeviceCredentials(TransportProtos.GetDeviceCredentialsRequestMsg requestMsg) { + TbProtoQueueMsg protoMsg = new TbProtoQueueMsg<>( + UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder() + .setDeviceCredentialsRequestMsg(requestMsg) + .build() + ); + + try { + TbProtoQueueMsg response = transportApiRequestTemplate.send(protoMsg).get(); + return response.getValue().getDeviceCredentialsResponseMsg(); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + } + @Override public void process(DeviceTransportType transportType, TransportProtos.ValidateDeviceTokenRequestMsg msg, TransportServiceCallback callback) { @@ -685,7 +741,10 @@ public class DefaultTransportService implements TransportService { } } else if (EntityType.DEVICE.equals(entityType)) { Optional deviceOpt = dataDecodingEncodingService.decode(msg.getData().toByteArray()); - deviceOpt.ifPresent(this::onDeviceUpdate); + deviceOpt.ifPresent(device -> { + onDeviceUpdate(device); + eventPublisher.publishEvent(new DeviceUpdatedEvent(device)); + }); } } else if (toSessionMsg.hasEntityDeleteMsg()) { TransportProtos.EntityDeleteMsg msg = toSessionMsg.getEntityDeleteMsg(); @@ -699,6 +758,7 @@ public class DefaultTransportService implements TransportService { rateLimitService.remove(new TenantId(entityUuid)); } else if (EntityType.DEVICE.equals(entityType)) { rateLimitService.remove(new DeviceId(entityUuid)); + onDeviceDeleted(new DeviceId(entityUuid)); } } else if (toSessionMsg.hasResourceUpdateMsg()) { //TODO: update resource cache @@ -764,6 +824,17 @@ public class DefaultTransportService implements TransportService { }); } + private void onDeviceDeleted(DeviceId deviceId) { + sessions.forEach((id, md) -> { + DeviceId sessionDeviceId = new DeviceId(new UUID(md.getSessionInfo().getDeviceIdMSB(), md.getSessionInfo().getDeviceIdLSB())); + if (sessionDeviceId.equals(deviceId)) { + transportCallbackExecutor.submit(() -> { + md.getListener().onDeviceDeleted(deviceId); + }); + } + }); + } + protected UUID toSessionId(TransportProtos.SessionInfoProto sessionInfo) { return new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 7f8d512c70..547725b635 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device; import com.google.common.util.concurrent.ListenableFuture; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -215,4 +216,7 @@ public interface DeviceDao extends Dao, TenantEntityDao { * @return the list of device objects */ PageData findDevicesByTenantIdAndProfileId(UUID tenantId, UUID profileId, PageLink pageLink); + + List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index f933cbd543..64b44db686 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -36,6 +36,7 @@ import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; import org.thingsboard.server.common.data.DeviceProfile; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; @@ -80,6 +81,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; @@ -554,6 +556,11 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe return savedDevice; } + @Override + public List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) { + return deviceDao.findDevicesIdsByDeviceProfileTransportType(transportType); + } + private DataValidator deviceValidator = new DataValidator() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index c39e481638..708e0f38e3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -20,6 +20,7 @@ import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.PagingAndSortingRepository; import org.springframework.data.repository.query.Param; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.dao.model.sql.DeviceEntity; import org.thingsboard.server.dao.model.sql.DeviceInfoEntity; @@ -170,4 +171,9 @@ public interface DeviceRepository extends PagingAndSortingRepository findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 18979eba91..cb91ffac4f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -22,6 +22,7 @@ import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; +import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; @@ -114,6 +115,11 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao DaoUtil.toPageable(pageLink))); } + @Override + public List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) { + return deviceRepository.findIdsByDeviceProfileTransportType(transportType); + } + @Override public PageData findDeviceInfosByTenantIdAndCustomerId(UUID tenantId, UUID customerId, PageLink pageLink) { return DaoUtil.toPageData( diff --git a/docker/tb-transports/snmp/conf/logback.xml b/docker/tb-transports/snmp/conf/logback.xml new file mode 100644 index 0000000000..e581a97c42 --- /dev/null +++ b/docker/tb-transports/snmp/conf/logback.xml @@ -0,0 +1,50 @@ + + + + + + + /var/log/tb-snmp-transport/${TB_SERVICE_ID}/tb-snmp-transport.log + + /var/log/tb-snmp-transport/${TB_SERVICE_ID}/tb-snmp-transport.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + \ No newline at end of file diff --git a/docker/tb-transports/snmp/conf/tb-snmp-transport.conf b/docker/tb-transports/snmp/conf/tb-snmp-transport.conf new file mode 100644 index 0000000000..2aceae797c --- /dev/null +++ b/docker/tb-transports/snmp/conf/tb-snmp-transport.conf @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/tb-snmp-transport/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/tb-snmp-transport/${TB_SERVICE_ID}-heapdump.bin" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export JAVA_OPTS="$JAVA_OPTS -XX:+ExitOnOutOfMemoryError" +export LOG_FILENAME=tb-snmp-transport.out +export LOADER_PATH=/usr/share/tb-snmp-transport/conf diff --git a/msa/transport/snmp/docker/Dockerfile b/msa/transport/snmp/docker/Dockerfile index e39624d538..b808932482 100644 --- a/msa/transport/snmp/docker/Dockerfile +++ b/msa/transport/snmp/docker/Dockerfile @@ -14,7 +14,7 @@ # limitations under the License. # -FROM thingsboard/openjdk8 +FROM thingsboard/openjdk11 COPY start-tb-snmp-transport.sh ${pkg.name}.deb /tmp/ diff --git a/transport/snmp/src/main/conf/logback.xml b/transport/snmp/src/main/conf/logback.xml new file mode 100644 index 0000000000..d9818c2257 --- /dev/null +++ b/transport/snmp/src/main/conf/logback.xml @@ -0,0 +1,45 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + diff --git a/transport/snmp/src/main/conf/tb-snmp-transport.conf b/transport/snmp/src/main/conf/tb-snmp-transport.conf new file mode 100644 index 0000000000..73b6e04f68 --- /dev/null +++ b/transport/snmp/src/main/conf/tb-snmp-transport.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/transport/snmp/src/main/resources/logback.xml b/transport/snmp/src/main/resources/logback.xml new file mode 100644 index 0000000000..7a3d9f6f0d --- /dev/null +++ b/transport/snmp/src/main/resources/logback.xml @@ -0,0 +1,36 @@ + + + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + \ No newline at end of file diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml new file mode 100644 index 0000000000..9f6bd131ac --- /dev/null +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -0,0 +1,394 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +server: + # Server bind address + address: "${HTTP_BIND_ADDRESS:0.0.0.0}" + # Server bind port + port: "${HTTP_BIND_PORT:8085}" + # Server SSL configuration + +# Zookeeper connection parameters. Used for service discovery. +zk: + # Enable/disable zookeeper discovery service. + enabled: "${ZOOKEEPER_ENABLED:false}" + # Zookeeper connect string + url: "${ZOOKEEPER_URL:localhost:2181}" + # Zookeeper retry interval in milliseconds + retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" + # Zookeeper connection timeout in milliseconds + connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" + # Zookeeper session timeout in milliseconds + session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" + # Name of the directory in zookeeper 'filesystem' + zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" + +cluster: + stats: + enabled: "${TB_CLUSTER_STATS_ENABLED:false}" + print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" + +cache: + # caffeine or redis + type: "${CACHE_TYPE:caffeine}" + attributes: + # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' + enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" + +caffeine: + specs: + relations: + timeToLiveInMinutes: 1440 + maxSize: 0 + deviceCredentials: + timeToLiveInMinutes: 1440 + maxSize: 0 + devices: + timeToLiveInMinutes: 1440 + maxSize: 0 + sessions: + timeToLiveInMinutes: 1440 + maxSize: 0 + assets: + timeToLiveInMinutes: 1440 + maxSize: 0 + entityViews: + timeToLiveInMinutes: 1440 + maxSize: 0 + claimDevices: + timeToLiveInMinutes: 1 + maxSize: 0 + securitySettings: + timeToLiveInMinutes: 1440 + maxSize: 0 + tenantProfiles: + timeToLiveInMinutes: 1440 + maxSize: 0 + deviceProfiles: + timeToLiveInMinutes: 1440 + maxSize: 0 + attributes: + timeToLiveInMinutes: 1440 + maxSize: 100000 + +redis: + # standalone or cluster + connection: + type: "${REDIS_CONNECTION_TYPE:standalone}" + standalone: + host: "${REDIS_HOST:localhost}" + port: "${REDIS_PORT:6379}" + useDefaultClientConfig: "${REDIS_USE_DEFAULT_CLIENT_CONFIG:true}" + # this value may be used only if you used not default ClientConfig + clientName: "${REDIS_CLIENT_NAME:standalone}" + # this value may be used only if you used not default ClientConfig + connectTimeout: "${REDIS_CLIENT_CONNECT_TIMEOUT:30000}" + # this value may be used only if you used not default ClientConfig + readTimeout: "${REDIS_CLIENT_READ_TIMEOUT:60000}" + # this value may be used only if you used not default ClientConfig + usePoolConfig: "${REDIS_CLIENT_USE_POOL_CONFIG:false}" + cluster: + # Comma-separated list of "host:port" pairs to bootstrap from. + nodes: "${REDIS_NODES:}" + # Maximum number of redirects to follow when executing commands across the cluster. + max-redirects: "${REDIS_MAX_REDIRECTS:12}" + useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" + # db index + db: "${REDIS_DB:0}" + # db password + password: "${REDIS_PASSWORD:}" + # pool config + pool_config: + maxTotal: "${REDIS_POOL_CONFIG_MAX_TOTAL:128}" + maxIdle: "${REDIS_POOL_CONFIG_MAX_IDLE:128}" + minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" + testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" + testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" + testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" + minEvictableMs: "${REDIS_POOL_CONFIG_MIN_EVICTABLE_MS:60000}" + evictionRunsMs: "${REDIS_POOL_CONFIG_EVICTION_RUNS_MS:30000}" + maxWaitMills: "${REDIS_POOL_CONFIG_MAX_WAIT_MS:60000}" + numberTestsPerEvictionRun: "${REDIS_POOL_CONFIG_NUMBER_TESTS_PER_EVICTION_RUN:3}" + blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" + +# Check new version updates parameters +updates: + # Enable/disable updates checking. + enabled: "${UPDATES_ENABLED:true}" + +# spring freemarker configuration +spring.freemarker.checkTemplateLocation: "false" + +audit-log: + # Enable/disable audit log functionality. + enabled: "${AUDIT_LOG_ENABLED:true}" + # Specify partitioning size for audit log by tenant id storage. Example MINUTES, HOURS, DAYS, MONTHS + by_tenant_partitioning: "${AUDIT_LOG_BY_TENANT_PARTITIONING:MONTHS}" + # Number of days as history period if startTime and endTime are not specified + default_query_period: "${AUDIT_LOG_DEFAULT_QUERY_PERIOD:30}" + # Logging levels per each entity type. + # Allowed values: OFF (disable), W (log write operations), RW (log read and write operations) + logging-level: + mask: + "device": "${AUDIT_LOG_MASK_DEVICE:W}" + "asset": "${AUDIT_LOG_MASK_ASSET:W}" + "dashboard": "${AUDIT_LOG_MASK_DASHBOARD:W}" + "customer": "${AUDIT_LOG_MASK_CUSTOMER:W}" + "user": "${AUDIT_LOG_MASK_USER:W}" + "rule_chain": "${AUDIT_LOG_MASK_RULE_CHAIN:W}" + "alarm": "${AUDIT_LOG_MASK_ALARM:W}" + "entity_view": "${AUDIT_LOG_MASK_ENTITY_VIEW:W}" + "device_profile": "${AUDIT_LOG_MASK_DEVICE_PROFILE:W}" + sink: + # Type of external sink. possible options: none, elasticsearch + type: "${AUDIT_LOG_SINK_TYPE:none}" + # Name of the index where audit logs stored + # Index name could contain next placeholders (not mandatory): + # @{TENANT} - substituted by tenant ID + # @{DATE} - substituted by current date in format provided in audit_log.sink.date_format + index_pattern: "${AUDIT_LOG_SINK_INDEX_PATTERN:@{TENANT}_AUDIT_LOG_@{DATE}}" + # Date format. Details of the pattern could be found here: + # https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html + date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.DD}" + scheme_name: "${AUDIT_LOG_SINK_SCHEME_NAME:http}" # http or https + host: "${AUDIT_LOG_SINK_HOST:localhost}" + port: "${AUDIT_LOG_SINK_PORT:9200}" + user_name: "${AUDIT_LOG_SINK_USER_NAME:}" + password: "${AUDIT_LOG_SINK_PASSWORD:}" + +state: + # Should be greater then transport.sessions.report_timeout + defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" + defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" + persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" + +transport: + sessions: + inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" + report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" + json: + # Cast String data types to Numeric if possible when processing Telemetry/Attributes JSON + type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" + # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) + max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" + # Enable/disable http/mqtt/coap transport protocols (has higher priority than certain protocol's 'enabled' property) + api_enabled: "${TB_TRANSPORT_API_ENABLED:true}" + # Local LwM2M transport parameters + snmp: + enabled: "${SNMP_ENABLED:true}" + +queue: + type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + kafka: + bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" + acks: "${TB_KAFKA_ACKS:all}" + retries: "${TB_KAFKA_RETRIES:1}" + batch.size: "${TB_KAFKA_BATCH_SIZE:16384}" + linger.ms: "${TB_KAFKA_LINGER_MS:1}" + buffer.memory: "${TB_BUFFER_MEMORY:33554432}" + replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" + max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" + max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" + fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + other: + topic-properties: + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" + consumer-stats: + enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" + kafka-response-timeout-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_RESPONSE_TIMEOUT_MS:1000}" + aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" + access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" + secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" + region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" + threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" + queue-properties: + rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds + pubsub: + project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" + service_account: "${TB_QUEUE_PUBSUB_SERVICE_ACCOUNT:YOUR_SERVICE_ACCOUNT}" + max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes + max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + service_bus: + namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" + sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" + sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" + max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" + queue-properties: + rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + rabbitmq: + exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" + host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" + port: "${TB_QUEUE_RABBIT_MQ_PORT:5672}" + virtual_host: "${TB_QUEUE_RABBIT_MQ_VIRTUAL_HOST:/}" + username: "${TB_QUEUE_RABBIT_MQ_USERNAME:YOUR_USERNAME}" + password: "${TB_QUEUE_RABBIT_MQ_PASSWORD:YOUR_PASSWORD}" + automatic_recovery_enabled: "${TB_QUEUE_RABBIT_MQ_AUTOMATIC_RECOVERY_ENABLED:false}" + connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" + handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" + queue-properties: + rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + partitions: + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + transport_api: + requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" + responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" + max_pending_requests: "${TB_QUEUE_TRANSPORT_MAX_PENDING_REQUESTS:10000}" + max_requests_timeout: "${TB_QUEUE_TRANSPORT_MAX_REQUEST_TIMEOUT:10000}" + max_callback_threads: "${TB_QUEUE_TRANSPORT_MAX_CALLBACK_THREADS:100}" + request_poll_interval: "${TB_QUEUE_TRANSPORT_REQUEST_POLL_INTERVAL_MS:25}" + response_poll_interval: "${TB_QUEUE_TRANSPORT_RESPONSE_POLL_INTERVAL_MS:25}" + core: + topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" + poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" + stats: + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + js: + # JS Eval request topic + request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" + # JS Eval responses topic prefix that is combined with node id + response_topic_prefix: "${REMOTE_JS_EVAL_RESPONSE_TOPIC:js_eval.responses}" + # JS Eval max pending requests + max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" + # JS Eval max request timeout + max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}" + # JS max request timeout + max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" + # JS response poll interval + response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" + rule-engine: + topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" + poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}" + stats: + enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" + print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" + queues: + - name: "${TB_QUEUE_RE_MAIN_QUEUE_NAME:Main}" + topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" + poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:2000}" + submit-strategy: + type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_BATCH_SIZE:1000}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_TYPE:SKIP_ALL_FAILURES}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; + max-pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:3}"# Max allowed time in seconds for pause between retries. + - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" + topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" + poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:2000}" + submit-strategy: + type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + max-pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries. + - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" + topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" + poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" + partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:2000}" + submit-strategy: + type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL + # For BATCH only + batch-size: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_BATCH_SIZE:100}" # Maximum number of messages in batch + processing-strategy: + type: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_TYPE:RETRY_FAILED_AND_TIMED_OUT}" # SKIP_ALL_FAILURES, RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + # For RETRY_ALL, RETRY_FAILED, RETRY_TIMED_OUT, RETRY_FAILED_AND_TIMED_OUT + retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited + failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; + pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; + max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries. + transport: + # For high priority notifications that require minimum latency and processing time + notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" + poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" + +event: + debug: + max-symbols: "${TB_MAX_DEBUG_EVENT_SYMBOLS:4096}" + +service: + type: "${TB_SERVICE_TYPE:tb-transport}" + # Unique id for this service (autogenerated if empty) + id: "${TB_SERVICE_ID:}" + tenant_id: "${TB_SERVICE_TENANT_ID:}" # empty or specific tenant id. + +metrics: + # Enable/disable actuator metrics. + enabled: "${METRICS_ENABLED:false}" + timer: + # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). + percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" + +management: + endpoints: + web: + exposure: + # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). + include: '${METRICS_ENDPOINTS_EXPOSE:info}' From 0532d22d9dfeede30fd206b41ea1398629c654e5 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 24 Mar 2021 12:05:43 +0200 Subject: [PATCH 04/32] Refactor --- ...ProfileKvMapping.java => SnmpMapping.java} | 9 +- .../SnmpProfileTransportConfiguration.java | 10 +- .../DefaultTbServiceInfoProvider.java | 5 +- .../server/queue/util/AfterContextReady.java | 35 +++ .../server/queue/util/AfterStartUp.java | 35 +++ .../transport/snmp/SnmpTransportContext.java | 31 +-- .../snmp/service/SnmpTransportService.java | 237 +++++++----------- 7 files changed, 184 insertions(+), 178 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/device/profile/{SnmpDeviceProfileKvMapping.java => SnmpMapping.java} (90%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java similarity index 90% rename from common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java rename to common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java index 6eff7cbd34..6bd0ca6b4d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java @@ -18,11 +18,10 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; import org.thingsboard.server.common.data.kv.DataType; -//TODO: rename class @Data -public class SnmpDeviceProfileKvMapping { - private String key; - private DataType type; - private String method; +public class SnmpMapping { private String oid; + private String method; + private String key; + private DataType dataType; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java index 50f6c4351c..fb6b46fbb6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java @@ -29,8 +29,8 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport private int pollPeriodMs; private int timeoutMs; private int retries; - private List attributes; - private List telemetry; + private List attributesMappings; + private List telemetryMappings; @Override public DeviceTransportType getType() { @@ -38,9 +38,9 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport } @JsonIgnore - public List getKvMappings() { - if (attributes != null && telemetry != null) { - return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList()); + public List getAllMappings() { + if (attributesMappings != null && telemetryMappings != null) { + return Stream.concat(attributesMappings.stream(), telemetryMappings.stream()).collect(Collectors.toList()); } else { return Collections.emptyList(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 3ba360cd8e..6988bef9e1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; -import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.TbTransportService; @@ -31,6 +29,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.util.AfterContextReady; import javax.annotation.PostConstruct; import java.net.InetAddress; @@ -109,7 +108,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { serviceInfo = builder.build(); } - @EventListener(ContextRefreshedEvent.class) + @AfterContextReady public void setTransports() { serviceInfo = ServiceInfo.newBuilder(serviceInfo) .addAllTransports(getTransportServices().stream() diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java new file mode 100644 index 0000000000..5c68a7609a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.AliasFor; +import org.springframework.core.annotation.Order; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@EventListener(ContextRefreshedEvent.class) +@Order +public @interface AfterContextReady { + @AliasFor(annotation = Order.class, attribute = "value") + int order() default Integer.MAX_VALUE; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java new file mode 100644 index 0000000000..5d2e94ee68 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.AliasFor; +import org.springframework.core.annotation.Order; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@EventListener(ApplicationReadyEvent.class) +@Order +public @interface AfterStartUp { + @AliasFor(annotation = Order.class, attribute = "value") + int order() default Integer.MAX_VALUE; +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index a9fa8380fa..df2582005a 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -20,16 +20,14 @@ import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.smi.OID; import org.snmp4j.smi.VariableBinding; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileKvMapping; +import org.thingsboard.server.common.data.device.profile.SnmpMapping; import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -45,6 +43,7 @@ import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsRes import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; import org.thingsboard.server.transport.snmp.service.ProtoTransportEntityService; import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService; @@ -61,7 +60,6 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; @TbSnmpTransportComponent @@ -80,8 +78,7 @@ public class SnmpTransportContext extends TransportContext { private final Map> profilesPdus = new ConcurrentHashMap<>(); private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); - @EventListener(ApplicationReadyEvent.class) - @Order(2) + @AfterStartUp(order = 2) public void initDevicesSessions() { log.info("Initializing SNMP devices sessions"); allSnmpDevicesIds = protoEntityService.getAllSnmpDevicesIds().stream() @@ -209,17 +206,17 @@ public class SnmpTransportContext extends TransportContext { } private List createPdus(SnmpProfileTransportConfiguration deviceProfileConfig) { - Map> varBindingPerMethod = new HashMap<>(); + Map> bindingsPerMethod = new HashMap<>(); - deviceProfileConfig.getKvMappings().forEach(mapping -> varBindingPerMethod + deviceProfileConfig.getAllMappings().forEach(mapping -> bindingsPerMethod .computeIfAbsent(mapping.getMethod(), v -> new ArrayList<>()) .add(new VariableBinding(new OID(mapping.getOid())))); - return varBindingPerMethod.keySet().stream() + return bindingsPerMethod.keySet().stream() .map(method -> { PDU request = new PDU(); request.setType(getSnmpMethod(method)); - request.addAll(varBindingPerMethod.get(method)); + request.addAll(bindingsPerMethod.get(method)); return request; }) .collect(Collectors.toList()); @@ -300,31 +297,27 @@ public class SnmpTransportContext extends TransportContext { return profilesPdus; } - public Optional getAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) { + public Optional getAttributeMapping(DeviceProfileId deviceProfileId, OID responseOid) { if (profilesTransportConfigs.containsKey(deviceProfileId)) { - return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributes()); + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributesMappings()); } return Optional.empty(); } - public Optional getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { + public Optional getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { if (profilesTransportConfigs.containsKey(deviceProfileId)) { - return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetry()); + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetryMappings()); } return Optional.empty(); } - private Optional getMapping(OID responseOid, List mappings) { + private Optional getMapping(OID responseOid, List mappings) { return mappings.stream() .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) //TODO: OID shouldn't be duplicated in the config, add backend and UI verification .findFirst(); } - public ExecutorService getSnmpCallbackExecutor() { - return snmpTransportService.getSnmpCallbackExecutor(); - } - private int getSnmpMethod(String configMethod) { switch (configMethod) { case "get": diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 22e60c98ee..3e9eb63943 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -15,114 +15,74 @@ */ package org.thingsboard.server.transport.snmp.service; -import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; -import lombok.AllArgsConstructor; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.Snmp; import org.snmp4j.event.ResponseEvent; import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultUdpTransportMapping; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.Lazy; -import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.auth.SessionInfoCreator; -import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.transport.snmp.SnmpTransportContext; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; +import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; @TbSnmpTransportComponent @Service @Slf4j public class SnmpTransportService implements TbTransportService { private final SnmpTransportContext snmpTransportContext; + private final TransportService transportService; - @Getter - private ExecutorService snmpCallbackExecutor; - @Getter private Snmp snmp; private ScheduledExecutorService pollingExecutor; + private ExecutorService snmpResponseProcessingExecutor; - public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext) { + public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext, + TransportService transportService) { this.snmpTransportContext = snmpTransportContext; + this.transportService = transportService; } - // @PostConstruct - private void init() { - log.info("Starting SNMP transport..."); + @PostConstruct + private void init() throws IOException { + log.info("Initializing SNMP transport service"); + pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); //TODO: Set parallelism value in the config - snmpCallbackExecutor = Executors.newWorkStealingPool(20); + snmpResponseProcessingExecutor = Executors.newWorkStealingPool(20); initializeSnmp(); - log.info("SNMP transport started!"); - } - - @PreDestroy - public void shutdown() { - log.info("Stopping SNMP transport!"); - if (pollingExecutor != null) { - pollingExecutor.shutdownNow(); - } - if (snmpCallbackExecutor != null) { - snmpCallbackExecutor.shutdownNow(); - } - if (snmp != null) { - try { - snmp.close(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - log.info("SNMP transport stopped!"); - } - - @EventListener(ApplicationReadyEvent.class) - @Order(value = 10) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting SNMP polling."); -// startPolling(); + log.info("SNMP transport service initialized"); } - private void initializeSnmp() { - try { - this.snmp = new Snmp(new DefaultUdpTransportMapping()); - this.snmp.listen(); - } catch (IOException e) { - //TODO: what should be done if transport wasn't initialized? - log.error(e.getMessage(), e); - } + private void initializeSnmp() throws IOException { + snmp = new Snmp(new DefaultUdpTransportMapping()); + snmp.listen(); } - private void startPolling() { + @AfterStartUp(order = 10) + public void startPolling() { + log.info("Starting SNMP polling"); //TODO: Get poll period from configuration; int pollPeriodSeconds = 1; @@ -141,7 +101,7 @@ public class SnmpTransportService implements TbTransportService { DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); snmpTransportContext.getProfilesPdus().get(deviceProfileId).forEach(pdu -> { try { - log.debug("[{}] Sending SNMP message...", pdu.getRequestID()); + log.debug("[{}] Sending SNMP message for device {}", pdu.getRequestID(), sessionContext.getDeviceId()); snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext); } catch (IOException e) { log.error(e.getMessage(), e); @@ -152,118 +112,84 @@ public class SnmpTransportService implements TbTransportService { public void onNewDeviceResponse(ResponseEvent responseEvent, DeviceSessionContext sessionContext) { ((Snmp) responseEvent.getSource()).cancel(responseEvent.getRequest(), sessionContext); - snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(responseEvent, sessionContext)); + snmpResponseProcessingExecutor.submit(() -> processSnmpResponse(responseEvent, sessionContext)); } private void processSnmpResponse(ResponseEvent event, DeviceSessionContext sessionContext) { - PDU response = event.getResponse(); if (event.getError() != null) { log.warn("Response error: {}", event.getError().getMessage(), event.getError()); + return; + } + + PDU response = event.getResponse(); + if (response == null) { + log.warn("No SNMP response, requestId: {}", event.getRequest().getRequestID()); + return; } - if (response != null) { - log.debug("[{}] Processing SNMP response: {}", response.getRequestID(), response); + DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); + log.debug("[{}] Processing SNMP response for device {} with device profile {}: {}", + response.getRequestID(), sessionContext.getDeviceId(), deviceProfileId, response); - DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); - TransportService transportService = snmpTransportContext.getTransportService(); - for (int i = 0; i < response.size(); i++) { - VariableBinding vb = response.get(i); - snmpTransportContext.getAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), - new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { - try { - transportService.process(sessionInfo, - convertToPostAttributes(kvMapping.getKey(), kvMapping.getType(), vb.toValueString()), - TransportServiceCallback.EMPTY); - reportActivity(sessionInfo); - } catch (Exception e) { - log.warn("Failed to process SNMP response: {}", e.getMessage(), e); - } - }))); - snmpTransportContext.getTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), - new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { - try { - transportService.process(sessionInfo, - convertToPostTelemetry(kvMapping.getKey(), kvMapping.getType(), vb.toValueString()), - TransportServiceCallback.EMPTY); - reportActivity(sessionInfo); + JsonObject telemetry = new JsonObject(); + JsonObject attributes = new JsonObject(); - } catch (Exception e) { - log.warn("Failed to process SNMP response: {}", e.getMessage(), e); - } - }))); - } - } else { - log.warn("No SNMP response, requestId: {}", event.getRequest().getRequestID()); + for (int i = 0; i < response.size(); i++) { + VariableBinding variableBinding = response.get(i); + log.trace("Processing variable binding {}: {}", i, variableBinding); + + snmpTransportContext.getTelemetryMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { + log.trace("Found telemetry mapping for oid {}: {}", variableBinding.getOid(), mapping); + processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), telemetry); + }); + + snmpTransportContext.getAttributeMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { + log.trace("Found attribute mapping for oid {}: {}", variableBinding.getOid(), mapping); + processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), attributes); + }); + } + + if (telemetry.entrySet().isEmpty() && attributes.entrySet().isEmpty()) { + log.warn("No telemetry or attribute values is the SNMP response for device {}", sessionContext.getDeviceId()); + return; } + + if (!telemetry.entrySet().isEmpty()) { + TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(telemetry); + transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, TransportServiceCallback.EMPTY); + log.debug("Posted telemetry for device {}: {}", sessionContext.getDeviceId(), telemetry); + } + + if (!attributes.entrySet().isEmpty()) { + TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(attributes); + transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, TransportServiceCallback.EMPTY); + log.debug("Posted attributes for device {}: {}", sessionContext.getDeviceId(), attributes); + } + + reportActivity(sessionContext.getSessionInfo()); } private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() + transportService.process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() .setAttributeSubscription(false) .setRpcSubscription(false) .setLastActivityTime(System.currentTimeMillis()) .build(), TransportServiceCallback.EMPTY); } - private TransportProtos.PostAttributeMsg convertToPostAttributes(String keyName, DataType dataType, String payload) throws AdaptorException { - try { - return JsonConverter.convertToAttributesProto(getKvJson(keyName, dataType, payload)); - } catch (IllegalStateException | JsonSyntaxException ex) { - //TODO: change the exception type - throw new AdaptorException(ex); - } - } - - private TransportProtos.PostTelemetryMsg convertToPostTelemetry(String keyName, DataType dataType, String payload) throws AdaptorException { - try { - return JsonConverter.convertToTelemetryProto(getKvJson(keyName, dataType, payload)); - } catch (IllegalStateException | JsonSyntaxException ex) { - //TODO: change the exception type - throw new AdaptorException(ex); - } - } - - private JsonElement getKvJson(String keyName, DataType dataType, String payload) throws AdaptorException { - JsonObject result = new JsonObject(); + private void processValue(String key, DataType dataType, String value, JsonObject result) { switch (dataType) { case LONG: - result.addProperty(keyName, Long.parseLong(payload)); + result.addProperty(key, Long.parseLong(value)); break; case BOOLEAN: - result.addProperty(keyName, Boolean.parseBoolean(payload)); + result.addProperty(key, Boolean.parseBoolean(value)); break; case DOUBLE: - result.addProperty(keyName, Double.parseDouble(payload)); - break; - case STRING: - result.addProperty(keyName, payload); + result.addProperty(key, Double.parseDouble(value)); break; default: - //TODO: change the exception type - throw new AdaptorException("Unsupported data type"); - } - return new JsonParser().parse(result.toString()); - } - - @AllArgsConstructor - private static class DeviceAuthCallback implements TransportServiceCallback { - private final TransportContext transportContext; - private final Consumer onSuccess; - - @Override - public void onSuccess(ValidateDeviceCredentialsResponse msg) { - if (msg.hasDeviceInfo()) { - onSuccess.accept(SessionInfoCreator.create(msg, transportContext, UUID.randomUUID())); - } else { - log.warn("Failed to process device auth"); - } - } - - @Override - public void onError(Throwable e) { - log.warn("Failed to process device auth", e); + result.addProperty(key, value); } } @@ -271,4 +197,23 @@ public class SnmpTransportService implements TbTransportService { public String getName() { return "SNMP"; } + + @PreDestroy + public void shutdown() { + log.info("Stopping SNMP transport!"); + if (pollingExecutor != null) { + pollingExecutor.shutdownNow(); + } + if (snmpResponseProcessingExecutor != null) { + snmpResponseProcessingExecutor.shutdownNow(); + } + if (snmp != null) { + try { + snmp.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + log.info("SNMP transport stopped!"); + } } From afab5150b8b39aac81a3c7f34a6f6fd438965d3b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 24 Mar 2021 16:43:28 +0200 Subject: [PATCH 05/32] Provide SNMP transport configuration validation; refactor --- .../data/DeviceTransportConfiguration.java | 3 + .../SnmpDeviceTransportConfiguration.java | 19 +++-- .../DeviceProfileTransportConfiguration.java | 6 +- ...pDeviceProfileTransportConfiguration.java} | 24 +++++-- .../data/transport/snmp/SnmpMapping.java | 39 ++++++++++ .../data/transport/snmp/SnmpMethod.java | 32 +++++++++ .../snmp/SnmpProtocolVersion.java} | 23 +++--- .../transport/snmp/SnmpTransportContext.java | 71 ++++++++----------- .../snmp/service/SnmpTransportService.java | 12 +++- .../snmp/session/DeviceSessionContext.java | 25 ++----- .../dao/device/DeviceProfileServiceImpl.java | 1 + .../server/dao/device/DeviceServiceImpl.java | 4 ++ 12 files changed, 173 insertions(+), 86 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/device/profile/{SnmpProfileTransportConfiguration.java => SnmpDeviceProfileTransportConfiguration.java} (60%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java rename common/data/src/main/java/org/thingsboard/server/common/data/{device/profile/SnmpMapping.java => transport/snmp/SnmpProtocolVersion.java} (68%) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java index 49a547371f..63b31eb3a2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java @@ -38,4 +38,7 @@ public interface DeviceTransportConfiguration extends Serializable { @JsonIgnore DeviceTransportType getType(); + default void validate() { + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index 3db31b4f2a..551bf2fcfb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -19,25 +19,30 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; @Data public class SnmpDeviceTransportConfiguration implements DeviceTransportConfiguration { - private String address; private int port; private String community; - private String protocolVersion; + private SnmpProtocolVersion protocolVersion; @Override public DeviceTransportType getType() { return DeviceTransportType.SNMP; } + @Override + public void validate() { + if (!isValid()) { + throw new IllegalArgumentException("Transport configuration is not valid"); + } + } + @JsonIgnore - public boolean isValid() { - return StringUtils.isNotEmpty(this.address) - && this.port > 0 - && StringUtils.isNotEmpty(this.community) - && StringUtils.isNotEmpty(this.protocolVersion); + private boolean isValid() { + return StringUtils.isNotBlank(address) && port > 0 && + StringUtils.isNotBlank(community) && protocolVersion != null; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java index ffbe5f7048..2fea8d34ad 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.exception.ThingsboardException; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo( @@ -30,10 +31,13 @@ import org.thingsboard.server.common.data.DeviceTransportType; @JsonSubTypes.Type(value = DefaultDeviceProfileTransportConfiguration.class, name = "DEFAULT"), @JsonSubTypes.Type(value = MqttDeviceProfileTransportConfiguration.class, name = "MQTT"), @JsonSubTypes.Type(value = Lwm2mDeviceProfileTransportConfiguration.class, name = "LWM2M"), - @JsonSubTypes.Type(value = SnmpProfileTransportConfiguration.class, name = "SNMP")}) + @JsonSubTypes.Type(value = SnmpDeviceProfileTransportConfiguration.class, name = "SNMP")}) public interface DeviceProfileTransportConfiguration { @JsonIgnore DeviceTransportType getType(); + default void validate() { + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java similarity index 60% rename from common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java index fb6b46fbb6..a4f7cc57de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; import java.util.Collections; import java.util.List; @@ -25,12 +26,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; @Data -public class SnmpProfileTransportConfiguration implements DeviceProfileTransportConfiguration { +public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTransportConfiguration { private int pollPeriodMs; private int timeoutMs; private int retries; - private List attributesMappings; private List telemetryMappings; + private List attributesMappings; @Override public DeviceTransportType getType() { @@ -39,10 +40,25 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport @JsonIgnore public List getAllMappings() { - if (attributesMappings != null && telemetryMappings != null) { - return Stream.concat(attributesMappings.stream(), telemetryMappings.stream()).collect(Collectors.toList()); + if (telemetryMappings != null && attributesMappings != null) { + return Stream.concat(telemetryMappings.stream(), attributesMappings.stream()).collect(Collectors.toList()); } else { return Collections.emptyList(); } } + + @Override + public void validate() { + if (!isValid()) { + throw new IllegalArgumentException("Transport configuration is not valid"); + } + } + + @JsonIgnore + private boolean isValid() { + List mappings = getAllMappings(); + return pollPeriodMs > 0 && timeoutMs > 0 && retries >= 0 && + !mappings.isEmpty() && mappings.stream().allMatch(SnmpMapping::isValid) && + mappings.stream().map(SnmpMapping::getOid).distinct().count() == mappings.size(); + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java new file mode 100644 index 0000000000..32a3f2a25b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.kv.DataType; + +import java.util.regex.Pattern; + +@Data +public class SnmpMapping { + private String oid; + private SnmpMethod method; + private String key; + private DataType dataType; + + private static final Pattern OID_PATTERN = Pattern.compile("^\\.?([0-2])((\\.0)|(\\.[1-9][0-9]*))*$"); + + @JsonIgnore + public boolean isValid() { + return StringUtils.isNotEmpty(oid) && OID_PATTERN.matcher(oid).matches() && method != null && + StringUtils.isNotBlank(key) && dataType != null; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java new file mode 100644 index 0000000000..2a04ce427b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp; + +public enum SnmpMethod { + GET(-96), + SET(-93); + + // codes taken from org.snmp4j.PDU class + private final int code; + + SnmpMethod(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpProtocolVersion.java similarity index 68% rename from common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java rename to common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpProtocolVersion.java index 6bd0ca6b4d..d16836cf1a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpProtocolVersion.java @@ -13,15 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.device.profile; +package org.thingsboard.server.common.data.transport.snmp; -import lombok.Data; -import org.thingsboard.server.common.data.kv.DataType; +public enum SnmpProtocolVersion { + V1(0), + V2C(1), + V3(3); -@Data -public class SnmpMapping { - private String oid; - private String method; - private String key; - private DataType dataType; + private final int code; + + SnmpProtocolVersion(int code) { + this.code = code; + } + + public int getCode() { + return code; + } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index df2582005a..beaaef829f 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -27,12 +27,13 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpMapping; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; import org.thingsboard.server.common.transport.DeviceUpdatedEvent; import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; @@ -74,7 +75,7 @@ public class SnmpTransportContext extends TransportContext { private final SnmpTransportBalancingService balancingService; private final Map sessions = new ConcurrentHashMap<>(); - private final Map profilesTransportConfigs = new ConcurrentHashMap<>(); + private final Map profilesTransportConfigs = new ConcurrentHashMap<>(); private final Map> profilesPdus = new ConcurrentHashMap<>(); private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); @@ -94,7 +95,13 @@ public class SnmpTransportContext extends TransportContext { managedDevicesIds.stream() .map(protoEntityService::getDeviceById) .collect(Collectors.toList()) - .forEach(this::establishDeviceSession); + .forEach(device -> { + try { + establishDeviceSession(device); + } catch (Exception e) { + log.error("Failed to establish session for SNMP device {}: {}", device.getId(), e.getMessage()); + } + }); } private void establishDeviceSession(Device device) { @@ -110,13 +117,10 @@ public class SnmpTransportContext extends TransportContext { return; } + SnmpDeviceProfileTransportConfiguration profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - if (!deviceTransportConfiguration.isValid()) { - log.warn("SNMP device transport configuration is not valid"); - return; - } - SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration); profilesPdus.computeIfAbsent(deviceProfileId, id -> createPdus(profileTransportConfiguration)); DeviceSessionContext deviceSessionContext = new DeviceSessionContext( @@ -140,22 +144,18 @@ public class SnmpTransportContext extends TransportContext { return; } - SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - sessionContext.setProfileTransportConfiguration(profileTransportConfiguration); - sessionContext.setDeviceTransportConfiguration(deviceTransportConfiguration); - if (!deviceTransportConfiguration.isValid()) { - log.warn("SNMP device transport configuration is not valid"); - destroyDeviceSession(sessionContext); - return; - } + SnmpDeviceProfileTransportConfiguration newProfileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + SnmpDeviceTransportConfiguration newDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + + if (!newProfileTransportConfiguration.equals(sessionContext.getProfileTransportConfiguration())) { + profilesPdus.put(deviceProfileId, createPdus(newProfileTransportConfiguration)); + profilesTransportConfigs.put(deviceProfileId, newProfileTransportConfiguration); - if (!profileTransportConfiguration.equals(profilesTransportConfigs.get(deviceProfileId))) { - profilesPdus.put(deviceProfileId, createPdus(profileTransportConfiguration)); - profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration); - sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); - } else if (!deviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { - sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); + sessionContext.setProfileTransportConfiguration(newProfileTransportConfiguration); + sessionContext.initTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); + } else if (!newDeviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { + sessionContext.setDeviceTransportConfiguration(newDeviceTransportConfiguration); + sessionContext.initTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); } else { log.trace("Configuration of the device {} was not updated", device); } @@ -205,8 +205,8 @@ public class SnmpTransportContext extends TransportContext { }); } - private List createPdus(SnmpProfileTransportConfiguration deviceProfileConfig) { - Map> bindingsPerMethod = new HashMap<>(); + private List createPdus(SnmpDeviceProfileTransportConfiguration deviceProfileConfig) { + Map> bindingsPerMethod = new HashMap<>(); deviceProfileConfig.getAllMappings().forEach(mapping -> bindingsPerMethod .computeIfAbsent(mapping.getMethod(), v -> new ArrayList<>()) @@ -215,7 +215,7 @@ public class SnmpTransportContext extends TransportContext { return bindingsPerMethod.keySet().stream() .map(method -> { PDU request = new PDU(); - request.setType(getSnmpMethod(method)); + request.setType(method.getCode()); request.addAll(bindingsPerMethod.get(method)); return request; }) @@ -311,22 +311,9 @@ public class SnmpTransportContext extends TransportContext { return Optional.empty(); } - private Optional getMapping(OID responseOid, List mappings) { + private Optional getMapping(OID oid, List mappings) { return mappings.stream() - .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) - //TODO: OID shouldn't be duplicated in the config, add backend and UI verification + .filter(mapping -> new OID(mapping.getOid()).equals(oid)) .findFirst(); } - - private int getSnmpMethod(String configMethod) { - switch (configMethod) { - case "get": - return PDU.GET; - case "getNext": - case "response": - case "set": - default: - return -1; - } - } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 3e9eb63943..86d2cead95 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.Snmp; import org.snmp4j.event.ResponseEvent; +import org.snmp4j.smi.Null; import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultUdpTransportMapping; import org.springframework.context.annotation.Lazy; @@ -103,8 +104,8 @@ public class SnmpTransportService implements TbTransportService { try { log.debug("[{}] Sending SNMP message for device {}", pdu.getRequestID(), sessionContext.getDeviceId()); snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext); - } catch (IOException e) { - log.error(e.getMessage(), e); + } catch (Exception e) { + log.error("Failed to send SNMP request: {}", e.getMessage()); } }); } @@ -123,7 +124,7 @@ public class SnmpTransportService implements TbTransportService { PDU response = event.getResponse(); if (response == null) { - log.warn("No SNMP response, requestId: {}", event.getRequest().getRequestID()); + log.warn("No response from SNMP device {}, requestId: {}", sessionContext.getDeviceId(), event.getRequest().getRequestID()); return; } @@ -138,6 +139,11 @@ public class SnmpTransportService implements TbTransportService { VariableBinding variableBinding = response.get(i); log.trace("Processing variable binding {}: {}", i, variableBinding); + if (variableBinding.getVariable() instanceof Null) { + log.debug("Response variable is empty"); + continue; + } + snmpTransportContext.getTelemetryMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { log.trace("Found telemetry mapping for oid {}: {}", variableBinding.getOid(), mapping); processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), telemetry); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index ed87bbc726..3cfc9bc041 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -22,13 +22,12 @@ import org.snmp4j.CommunityTarget; import org.snmp4j.Target; import org.snmp4j.event.ResponseEvent; import org.snmp4j.event.ResponseListener; -import org.snmp4j.mp.SnmpConstants; import org.snmp4j.smi.GenericAddress; import org.snmp4j.smi.OctetString; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; @@ -51,7 +50,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S private final String token; @Getter @Setter - private SnmpProfileTransportConfiguration profileTransportConfiguration; + private SnmpDeviceProfileTransportConfiguration profileTransportConfiguration; @Getter @Setter private SnmpDeviceTransportConfiguration deviceTransportConfiguration; @@ -79,7 +78,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S this.snmpTransportContext = snmpTransportContext; this.snmpTransportService = snmpTransportService; - this.profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + this.profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); this.deviceTransportConfiguration = deviceTransportConfiguration; initTarget(this.profileTransportConfiguration, this.deviceTransportConfiguration); @@ -105,11 +104,11 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S } } - public void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + public void initTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { log.trace("Initializing target for SNMP session of device {}", device); CommunityTarget communityTarget = new CommunityTarget(); communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); - communityTarget.setVersion(getSnmpVersion(deviceTransportConfig.getProtocolVersion())); + communityTarget.setVersion(deviceTransportConfig.getProtocolVersion().getCode()); communityTarget.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); communityTarget.setRetries(profileTransportConfig.getRetries()); @@ -125,20 +124,6 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S return token; } - //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer) - private int getSnmpVersion(String configSnmpVersion) { - switch (configSnmpVersion) { - case ("v1"): - return SnmpConstants.version1; - case ("v2c"): - return SnmpConstants.version2c; - case ("v3"): - return SnmpConstants.version3; - default: - return -1; - } - } - @Override public int nextMsgId() { return msgIdSeq.incrementAndGet(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 95f52cf55e..fea97bc903 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -350,6 +350,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D } DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + transportConfiguration.validate(); if (transportConfiguration instanceof MqttDeviceProfileTransportConfiguration) { MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; if (mqttTransportConfiguration.getTransportPayloadTypeConfiguration() instanceof ProtoTransportPayloadConfiguration) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 64b44db686..f3644d80ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.device.credentials.BasicMqttCredential import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.MqttDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; @@ -604,6 +605,9 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe throw new DataValidationException("Can't assign device to customer from different tenant!"); } } + Optional.ofNullable(device.getDeviceData()) + .flatMap(deviceData -> Optional.ofNullable(deviceData.getTransportConfiguration())) + .ifPresent(DeviceTransportConfiguration::validate); } }; From 039508bfbd3d155a5091a130dd4f1701bba9c223 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 24 Mar 2021 12:05:43 +0200 Subject: [PATCH 06/32] Refactor --- ...ProfileKvMapping.java => SnmpMapping.java} | 9 +- .../SnmpProfileTransportConfiguration.java | 10 +- .../DefaultTbServiceInfoProvider.java | 5 +- .../server/queue/util/AfterContextReady.java | 35 +++ .../server/queue/util/AfterStartUp.java | 35 +++ .../transport/snmp/SnmpTransportContext.java | 31 +-- .../snmp/service/SnmpTransportService.java | 237 +++++++----------- 7 files changed, 184 insertions(+), 178 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/device/profile/{SnmpDeviceProfileKvMapping.java => SnmpMapping.java} (90%) create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java create mode 100644 common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java similarity index 90% rename from common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java rename to common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java index 6eff7cbd34..6bd0ca6b4d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileKvMapping.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java @@ -18,11 +18,10 @@ package org.thingsboard.server.common.data.device.profile; import lombok.Data; import org.thingsboard.server.common.data.kv.DataType; -//TODO: rename class @Data -public class SnmpDeviceProfileKvMapping { - private String key; - private DataType type; - private String method; +public class SnmpMapping { private String oid; + private String method; + private String key; + private DataType dataType; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java index 50f6c4351c..fb6b46fbb6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java @@ -29,8 +29,8 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport private int pollPeriodMs; private int timeoutMs; private int retries; - private List attributes; - private List telemetry; + private List attributesMappings; + private List telemetryMappings; @Override public DeviceTransportType getType() { @@ -38,9 +38,9 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport } @JsonIgnore - public List getKvMappings() { - if (attributes != null && telemetry != null) { - return Stream.concat(attributes.stream(), telemetry.stream()).collect(Collectors.toList()); + public List getAllMappings() { + if (attributesMappings != null && telemetryMappings != null) { + return Stream.concat(attributesMappings.stream(), telemetryMappings.stream()).collect(Collectors.toList()); } else { return Collections.emptyList(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java index 3ba360cd8e..6988bef9e1 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/DefaultTbServiceInfoProvider.java @@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationContext; -import org.springframework.context.event.ContextRefreshedEvent; -import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.TbTransportService; @@ -31,6 +29,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.ServiceInfo; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbRuleEngineQueueConfiguration; +import org.thingsboard.server.queue.util.AfterContextReady; import javax.annotation.PostConstruct; import java.net.InetAddress; @@ -109,7 +108,7 @@ public class DefaultTbServiceInfoProvider implements TbServiceInfoProvider { serviceInfo = builder.build(); } - @EventListener(ContextRefreshedEvent.class) + @AfterContextReady public void setTransports() { serviceInfo = ServiceInfo.newBuilder(serviceInfo) .addAllTransports(getTransportServices().stream() diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java new file mode 100644 index 0000000000..5c68a7609a --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterContextReady.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.AliasFor; +import org.springframework.core.annotation.Order; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@EventListener(ContextRefreshedEvent.class) +@Order +public @interface AfterContextReady { + @AliasFor(annotation = Order.class, attribute = "value") + int order() default Integer.MAX_VALUE; +} diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java new file mode 100644 index 0000000000..5d2e94ee68 --- /dev/null +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/AfterStartUp.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.queue.util; + +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.core.annotation.AliasFor; +import org.springframework.core.annotation.Order; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@EventListener(ApplicationReadyEvent.class) +@Order +public @interface AfterStartUp { + @AliasFor(annotation = Order.class, attribute = "value") + int order() default Integer.MAX_VALUE; +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index a9fa8380fa..df2582005a 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -20,16 +20,14 @@ import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.smi.OID; import org.snmp4j.smi.VariableBinding; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileKvMapping; +import org.thingsboard.server.common.data.device.profile.SnmpMapping; import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -45,6 +43,7 @@ import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsRes import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; import org.thingsboard.server.transport.snmp.service.ProtoTransportEntityService; import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService; @@ -61,7 +60,6 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.concurrent.ExecutorService; import java.util.stream.Collectors; @TbSnmpTransportComponent @@ -80,8 +78,7 @@ public class SnmpTransportContext extends TransportContext { private final Map> profilesPdus = new ConcurrentHashMap<>(); private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); - @EventListener(ApplicationReadyEvent.class) - @Order(2) + @AfterStartUp(order = 2) public void initDevicesSessions() { log.info("Initializing SNMP devices sessions"); allSnmpDevicesIds = protoEntityService.getAllSnmpDevicesIds().stream() @@ -209,17 +206,17 @@ public class SnmpTransportContext extends TransportContext { } private List createPdus(SnmpProfileTransportConfiguration deviceProfileConfig) { - Map> varBindingPerMethod = new HashMap<>(); + Map> bindingsPerMethod = new HashMap<>(); - deviceProfileConfig.getKvMappings().forEach(mapping -> varBindingPerMethod + deviceProfileConfig.getAllMappings().forEach(mapping -> bindingsPerMethod .computeIfAbsent(mapping.getMethod(), v -> new ArrayList<>()) .add(new VariableBinding(new OID(mapping.getOid())))); - return varBindingPerMethod.keySet().stream() + return bindingsPerMethod.keySet().stream() .map(method -> { PDU request = new PDU(); request.setType(getSnmpMethod(method)); - request.addAll(varBindingPerMethod.get(method)); + request.addAll(bindingsPerMethod.get(method)); return request; }) .collect(Collectors.toList()); @@ -300,31 +297,27 @@ public class SnmpTransportContext extends TransportContext { return profilesPdus; } - public Optional getAttributesMapping(DeviceProfileId deviceProfileId, OID responseOid) { + public Optional getAttributeMapping(DeviceProfileId deviceProfileId, OID responseOid) { if (profilesTransportConfigs.containsKey(deviceProfileId)) { - return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributes()); + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributesMappings()); } return Optional.empty(); } - public Optional getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { + public Optional getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { if (profilesTransportConfigs.containsKey(deviceProfileId)) { - return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetry()); + return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetryMappings()); } return Optional.empty(); } - private Optional getMapping(OID responseOid, List mappings) { + private Optional getMapping(OID responseOid, List mappings) { return mappings.stream() .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) //TODO: OID shouldn't be duplicated in the config, add backend and UI verification .findFirst(); } - public ExecutorService getSnmpCallbackExecutor() { - return snmpTransportService.getSnmpCallbackExecutor(); - } - private int getSnmpMethod(String configMethod) { switch (configMethod) { case "get": diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 22e60c98ee..3e9eb63943 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -15,114 +15,74 @@ */ package org.thingsboard.server.transport.snmp.service; -import com.google.gson.JsonElement; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; -import lombok.AllArgsConstructor; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.Snmp; import org.snmp4j.event.ResponseEvent; import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultUdpTransportMapping; -import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.annotation.Lazy; -import org.springframework.context.event.EventListener; -import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; -import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; -import org.thingsboard.server.common.transport.auth.SessionInfoCreator; -import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.transport.snmp.SnmpTransportContext; +import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.SnmpTransportContext; import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; +import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; @TbSnmpTransportComponent @Service @Slf4j public class SnmpTransportService implements TbTransportService { private final SnmpTransportContext snmpTransportContext; + private final TransportService transportService; - @Getter - private ExecutorService snmpCallbackExecutor; - @Getter private Snmp snmp; private ScheduledExecutorService pollingExecutor; + private ExecutorService snmpResponseProcessingExecutor; - public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext) { + public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext, + TransportService transportService) { this.snmpTransportContext = snmpTransportContext; + this.transportService = transportService; } - // @PostConstruct - private void init() { - log.info("Starting SNMP transport..."); + @PostConstruct + private void init() throws IOException { + log.info("Initializing SNMP transport service"); + pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); //TODO: Set parallelism value in the config - snmpCallbackExecutor = Executors.newWorkStealingPool(20); + snmpResponseProcessingExecutor = Executors.newWorkStealingPool(20); initializeSnmp(); - log.info("SNMP transport started!"); - } - - @PreDestroy - public void shutdown() { - log.info("Stopping SNMP transport!"); - if (pollingExecutor != null) { - pollingExecutor.shutdownNow(); - } - if (snmpCallbackExecutor != null) { - snmpCallbackExecutor.shutdownNow(); - } - if (snmp != null) { - try { - snmp.close(); - } catch (IOException e) { - log.error(e.getMessage(), e); - } - } - log.info("SNMP transport stopped!"); - } - - @EventListener(ApplicationReadyEvent.class) - @Order(value = 10) - public void onApplicationEvent(ApplicationReadyEvent applicationReadyEvent) { - log.info("Received application ready event. Starting SNMP polling."); -// startPolling(); + log.info("SNMP transport service initialized"); } - private void initializeSnmp() { - try { - this.snmp = new Snmp(new DefaultUdpTransportMapping()); - this.snmp.listen(); - } catch (IOException e) { - //TODO: what should be done if transport wasn't initialized? - log.error(e.getMessage(), e); - } + private void initializeSnmp() throws IOException { + snmp = new Snmp(new DefaultUdpTransportMapping()); + snmp.listen(); } - private void startPolling() { + @AfterStartUp(order = 10) + public void startPolling() { + log.info("Starting SNMP polling"); //TODO: Get poll period from configuration; int pollPeriodSeconds = 1; @@ -141,7 +101,7 @@ public class SnmpTransportService implements TbTransportService { DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); snmpTransportContext.getProfilesPdus().get(deviceProfileId).forEach(pdu -> { try { - log.debug("[{}] Sending SNMP message...", pdu.getRequestID()); + log.debug("[{}] Sending SNMP message for device {}", pdu.getRequestID(), sessionContext.getDeviceId()); snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext); } catch (IOException e) { log.error(e.getMessage(), e); @@ -152,118 +112,84 @@ public class SnmpTransportService implements TbTransportService { public void onNewDeviceResponse(ResponseEvent responseEvent, DeviceSessionContext sessionContext) { ((Snmp) responseEvent.getSource()).cancel(responseEvent.getRequest(), sessionContext); - snmpTransportContext.getSnmpCallbackExecutor().submit(() -> processSnmpResponse(responseEvent, sessionContext)); + snmpResponseProcessingExecutor.submit(() -> processSnmpResponse(responseEvent, sessionContext)); } private void processSnmpResponse(ResponseEvent event, DeviceSessionContext sessionContext) { - PDU response = event.getResponse(); if (event.getError() != null) { log.warn("Response error: {}", event.getError().getMessage(), event.getError()); + return; + } + + PDU response = event.getResponse(); + if (response == null) { + log.warn("No SNMP response, requestId: {}", event.getRequest().getRequestID()); + return; } - if (response != null) { - log.debug("[{}] Processing SNMP response: {}", response.getRequestID(), response); + DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); + log.debug("[{}] Processing SNMP response for device {} with device profile {}: {}", + response.getRequestID(), sessionContext.getDeviceId(), deviceProfileId, response); - DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); - TransportService transportService = snmpTransportContext.getTransportService(); - for (int i = 0; i < response.size(); i++) { - VariableBinding vb = response.get(i); - snmpTransportContext.getAttributesMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), - new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { - try { - transportService.process(sessionInfo, - convertToPostAttributes(kvMapping.getKey(), kvMapping.getType(), vb.toValueString()), - TransportServiceCallback.EMPTY); - reportActivity(sessionInfo); - } catch (Exception e) { - log.warn("Failed to process SNMP response: {}", e.getMessage(), e); - } - }))); - snmpTransportContext.getTelemetryMapping(deviceProfileId, vb.getOid()).ifPresent(kvMapping -> transportService.process(DeviceTransportType.DEFAULT, - TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(sessionContext.getToken()).build(), - new DeviceAuthCallback(snmpTransportContext, sessionInfo -> { - try { - transportService.process(sessionInfo, - convertToPostTelemetry(kvMapping.getKey(), kvMapping.getType(), vb.toValueString()), - TransportServiceCallback.EMPTY); - reportActivity(sessionInfo); + JsonObject telemetry = new JsonObject(); + JsonObject attributes = new JsonObject(); - } catch (Exception e) { - log.warn("Failed to process SNMP response: {}", e.getMessage(), e); - } - }))); - } - } else { - log.warn("No SNMP response, requestId: {}", event.getRequest().getRequestID()); + for (int i = 0; i < response.size(); i++) { + VariableBinding variableBinding = response.get(i); + log.trace("Processing variable binding {}: {}", i, variableBinding); + + snmpTransportContext.getTelemetryMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { + log.trace("Found telemetry mapping for oid {}: {}", variableBinding.getOid(), mapping); + processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), telemetry); + }); + + snmpTransportContext.getAttributeMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { + log.trace("Found attribute mapping for oid {}: {}", variableBinding.getOid(), mapping); + processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), attributes); + }); + } + + if (telemetry.entrySet().isEmpty() && attributes.entrySet().isEmpty()) { + log.warn("No telemetry or attribute values is the SNMP response for device {}", sessionContext.getDeviceId()); + return; } + + if (!telemetry.entrySet().isEmpty()) { + TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(telemetry); + transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, TransportServiceCallback.EMPTY); + log.debug("Posted telemetry for device {}: {}", sessionContext.getDeviceId(), telemetry); + } + + if (!attributes.entrySet().isEmpty()) { + TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(attributes); + transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, TransportServiceCallback.EMPTY); + log.debug("Posted attributes for device {}: {}", sessionContext.getDeviceId(), attributes); + } + + reportActivity(sessionContext.getSessionInfo()); } private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { - snmpTransportContext.getTransportService().process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() + transportService.process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() .setAttributeSubscription(false) .setRpcSubscription(false) .setLastActivityTime(System.currentTimeMillis()) .build(), TransportServiceCallback.EMPTY); } - private TransportProtos.PostAttributeMsg convertToPostAttributes(String keyName, DataType dataType, String payload) throws AdaptorException { - try { - return JsonConverter.convertToAttributesProto(getKvJson(keyName, dataType, payload)); - } catch (IllegalStateException | JsonSyntaxException ex) { - //TODO: change the exception type - throw new AdaptorException(ex); - } - } - - private TransportProtos.PostTelemetryMsg convertToPostTelemetry(String keyName, DataType dataType, String payload) throws AdaptorException { - try { - return JsonConverter.convertToTelemetryProto(getKvJson(keyName, dataType, payload)); - } catch (IllegalStateException | JsonSyntaxException ex) { - //TODO: change the exception type - throw new AdaptorException(ex); - } - } - - private JsonElement getKvJson(String keyName, DataType dataType, String payload) throws AdaptorException { - JsonObject result = new JsonObject(); + private void processValue(String key, DataType dataType, String value, JsonObject result) { switch (dataType) { case LONG: - result.addProperty(keyName, Long.parseLong(payload)); + result.addProperty(key, Long.parseLong(value)); break; case BOOLEAN: - result.addProperty(keyName, Boolean.parseBoolean(payload)); + result.addProperty(key, Boolean.parseBoolean(value)); break; case DOUBLE: - result.addProperty(keyName, Double.parseDouble(payload)); - break; - case STRING: - result.addProperty(keyName, payload); + result.addProperty(key, Double.parseDouble(value)); break; default: - //TODO: change the exception type - throw new AdaptorException("Unsupported data type"); - } - return new JsonParser().parse(result.toString()); - } - - @AllArgsConstructor - private static class DeviceAuthCallback implements TransportServiceCallback { - private final TransportContext transportContext; - private final Consumer onSuccess; - - @Override - public void onSuccess(ValidateDeviceCredentialsResponse msg) { - if (msg.hasDeviceInfo()) { - onSuccess.accept(SessionInfoCreator.create(msg, transportContext, UUID.randomUUID())); - } else { - log.warn("Failed to process device auth"); - } - } - - @Override - public void onError(Throwable e) { - log.warn("Failed to process device auth", e); + result.addProperty(key, value); } } @@ -271,4 +197,23 @@ public class SnmpTransportService implements TbTransportService { public String getName() { return "SNMP"; } + + @PreDestroy + public void shutdown() { + log.info("Stopping SNMP transport!"); + if (pollingExecutor != null) { + pollingExecutor.shutdownNow(); + } + if (snmpResponseProcessingExecutor != null) { + snmpResponseProcessingExecutor.shutdownNow(); + } + if (snmp != null) { + try { + snmp.close(); + } catch (IOException e) { + log.error(e.getMessage(), e); + } + } + log.info("SNMP transport stopped!"); + } } From fc06e8ac998ae426af4a21b0fe713b5f9a380285 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 24 Mar 2021 16:43:28 +0200 Subject: [PATCH 07/32] Provide SNMP transport configuration validation; refactor --- .../data/DeviceTransportConfiguration.java | 3 + .../SnmpDeviceTransportConfiguration.java | 19 +++-- .../DeviceProfileTransportConfiguration.java | 6 +- ...pDeviceProfileTransportConfiguration.java} | 24 +++++-- .../data/transport/snmp/SnmpMapping.java | 39 ++++++++++ .../data/transport/snmp/SnmpMethod.java | 32 +++++++++ .../snmp/SnmpProtocolVersion.java} | 23 +++--- .../transport/snmp/SnmpTransportContext.java | 71 ++++++++----------- .../snmp/service/SnmpTransportService.java | 12 +++- .../snmp/session/DeviceSessionContext.java | 25 ++----- .../dao/device/DeviceProfileServiceImpl.java | 1 + .../server/dao/device/DeviceServiceImpl.java | 4 ++ 12 files changed, 173 insertions(+), 86 deletions(-) rename common/data/src/main/java/org/thingsboard/server/common/data/device/profile/{SnmpProfileTransportConfiguration.java => SnmpDeviceProfileTransportConfiguration.java} (60%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java rename common/data/src/main/java/org/thingsboard/server/common/data/{device/profile/SnmpMapping.java => transport/snmp/SnmpProtocolVersion.java} (68%) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java index 49a547371f..63b31eb3a2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/DeviceTransportConfiguration.java @@ -38,4 +38,7 @@ public interface DeviceTransportConfiguration extends Serializable { @JsonIgnore DeviceTransportType getType(); + default void validate() { + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index 3db31b4f2a..551bf2fcfb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -19,25 +19,30 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; @Data public class SnmpDeviceTransportConfiguration implements DeviceTransportConfiguration { - private String address; private int port; private String community; - private String protocolVersion; + private SnmpProtocolVersion protocolVersion; @Override public DeviceTransportType getType() { return DeviceTransportType.SNMP; } + @Override + public void validate() { + if (!isValid()) { + throw new IllegalArgumentException("Transport configuration is not valid"); + } + } + @JsonIgnore - public boolean isValid() { - return StringUtils.isNotEmpty(this.address) - && this.port > 0 - && StringUtils.isNotEmpty(this.community) - && StringUtils.isNotEmpty(this.protocolVersion); + private boolean isValid() { + return StringUtils.isNotBlank(address) && port > 0 && + StringUtils.isNotBlank(community) && protocolVersion != null; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java index ffbe5f7048..2fea8d34ad 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/DeviceProfileTransportConfiguration.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.exception.ThingsboardException; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo( @@ -30,10 +31,13 @@ import org.thingsboard.server.common.data.DeviceTransportType; @JsonSubTypes.Type(value = DefaultDeviceProfileTransportConfiguration.class, name = "DEFAULT"), @JsonSubTypes.Type(value = MqttDeviceProfileTransportConfiguration.class, name = "MQTT"), @JsonSubTypes.Type(value = Lwm2mDeviceProfileTransportConfiguration.class, name = "LWM2M"), - @JsonSubTypes.Type(value = SnmpProfileTransportConfiguration.class, name = "SNMP")}) + @JsonSubTypes.Type(value = SnmpDeviceProfileTransportConfiguration.class, name = "SNMP")}) public interface DeviceProfileTransportConfiguration { @JsonIgnore DeviceTransportType getType(); + default void validate() { + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java similarity index 60% rename from common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java rename to common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java index fb6b46fbb6..a4f7cc57de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data.device.profile; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; import java.util.Collections; import java.util.List; @@ -25,12 +26,12 @@ import java.util.stream.Collectors; import java.util.stream.Stream; @Data -public class SnmpProfileTransportConfiguration implements DeviceProfileTransportConfiguration { +public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTransportConfiguration { private int pollPeriodMs; private int timeoutMs; private int retries; - private List attributesMappings; private List telemetryMappings; + private List attributesMappings; @Override public DeviceTransportType getType() { @@ -39,10 +40,25 @@ public class SnmpProfileTransportConfiguration implements DeviceProfileTransport @JsonIgnore public List getAllMappings() { - if (attributesMappings != null && telemetryMappings != null) { - return Stream.concat(attributesMappings.stream(), telemetryMappings.stream()).collect(Collectors.toList()); + if (telemetryMappings != null && attributesMappings != null) { + return Stream.concat(telemetryMappings.stream(), attributesMappings.stream()).collect(Collectors.toList()); } else { return Collections.emptyList(); } } + + @Override + public void validate() { + if (!isValid()) { + throw new IllegalArgumentException("Transport configuration is not valid"); + } + } + + @JsonIgnore + private boolean isValid() { + List mappings = getAllMappings(); + return pollPeriodMs > 0 && timeoutMs > 0 && retries >= 0 && + !mappings.isEmpty() && mappings.stream().allMatch(SnmpMapping::isValid) && + mappings.stream().map(SnmpMapping::getOid).distinct().count() == mappings.size(); + } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java new file mode 100644 index 0000000000..32a3f2a25b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; +import org.thingsboard.server.common.data.kv.DataType; + +import java.util.regex.Pattern; + +@Data +public class SnmpMapping { + private String oid; + private SnmpMethod method; + private String key; + private DataType dataType; + + private static final Pattern OID_PATTERN = Pattern.compile("^\\.?([0-2])((\\.0)|(\\.[1-9][0-9]*))*$"); + + @JsonIgnore + public boolean isValid() { + return StringUtils.isNotEmpty(oid) && OID_PATTERN.matcher(oid).matches() && method != null && + StringUtils.isNotBlank(key) && dataType != null; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java new file mode 100644 index 0000000000..2a04ce427b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMethod.java @@ -0,0 +1,32 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp; + +public enum SnmpMethod { + GET(-96), + SET(-93); + + // codes taken from org.snmp4j.PDU class + private final int code; + + SnmpMethod(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpProtocolVersion.java similarity index 68% rename from common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java rename to common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpProtocolVersion.java index 6bd0ca6b4d..d16836cf1a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpMapping.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpProtocolVersion.java @@ -13,15 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.device.profile; +package org.thingsboard.server.common.data.transport.snmp; -import lombok.Data; -import org.thingsboard.server.common.data.kv.DataType; +public enum SnmpProtocolVersion { + V1(0), + V2C(1), + V3(3); -@Data -public class SnmpMapping { - private String oid; - private String method; - private String key; - private DataType dataType; + private final int code; + + SnmpProtocolVersion(int code) { + this.code = code; + } + + public int getCode() { + return code; + } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index df2582005a..beaaef829f 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -27,12 +27,13 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpMapping; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; import org.thingsboard.server.common.transport.DeviceUpdatedEvent; import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; @@ -74,7 +75,7 @@ public class SnmpTransportContext extends TransportContext { private final SnmpTransportBalancingService balancingService; private final Map sessions = new ConcurrentHashMap<>(); - private final Map profilesTransportConfigs = new ConcurrentHashMap<>(); + private final Map profilesTransportConfigs = new ConcurrentHashMap<>(); private final Map> profilesPdus = new ConcurrentHashMap<>(); private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); @@ -94,7 +95,13 @@ public class SnmpTransportContext extends TransportContext { managedDevicesIds.stream() .map(protoEntityService::getDeviceById) .collect(Collectors.toList()) - .forEach(this::establishDeviceSession); + .forEach(device -> { + try { + establishDeviceSession(device); + } catch (Exception e) { + log.error("Failed to establish session for SNMP device {}: {}", device.getId(), e.getMessage()); + } + }); } private void establishDeviceSession(Device device) { @@ -110,13 +117,10 @@ public class SnmpTransportContext extends TransportContext { return; } + SnmpDeviceProfileTransportConfiguration profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - if (!deviceTransportConfiguration.isValid()) { - log.warn("SNMP device transport configuration is not valid"); - return; - } - SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration); profilesPdus.computeIfAbsent(deviceProfileId, id -> createPdus(profileTransportConfiguration)); DeviceSessionContext deviceSessionContext = new DeviceSessionContext( @@ -140,22 +144,18 @@ public class SnmpTransportContext extends TransportContext { return; } - SnmpProfileTransportConfiguration profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); - SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - sessionContext.setProfileTransportConfiguration(profileTransportConfiguration); - sessionContext.setDeviceTransportConfiguration(deviceTransportConfiguration); - if (!deviceTransportConfiguration.isValid()) { - log.warn("SNMP device transport configuration is not valid"); - destroyDeviceSession(sessionContext); - return; - } + SnmpDeviceProfileTransportConfiguration newProfileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + SnmpDeviceTransportConfiguration newDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); + + if (!newProfileTransportConfiguration.equals(sessionContext.getProfileTransportConfiguration())) { + profilesPdus.put(deviceProfileId, createPdus(newProfileTransportConfiguration)); + profilesTransportConfigs.put(deviceProfileId, newProfileTransportConfiguration); - if (!profileTransportConfiguration.equals(profilesTransportConfigs.get(deviceProfileId))) { - profilesPdus.put(deviceProfileId, createPdus(profileTransportConfiguration)); - profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration); - sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); - } else if (!deviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { - sessionContext.initTarget(profileTransportConfiguration, deviceTransportConfiguration); + sessionContext.setProfileTransportConfiguration(newProfileTransportConfiguration); + sessionContext.initTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); + } else if (!newDeviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { + sessionContext.setDeviceTransportConfiguration(newDeviceTransportConfiguration); + sessionContext.initTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); } else { log.trace("Configuration of the device {} was not updated", device); } @@ -205,8 +205,8 @@ public class SnmpTransportContext extends TransportContext { }); } - private List createPdus(SnmpProfileTransportConfiguration deviceProfileConfig) { - Map> bindingsPerMethod = new HashMap<>(); + private List createPdus(SnmpDeviceProfileTransportConfiguration deviceProfileConfig) { + Map> bindingsPerMethod = new HashMap<>(); deviceProfileConfig.getAllMappings().forEach(mapping -> bindingsPerMethod .computeIfAbsent(mapping.getMethod(), v -> new ArrayList<>()) @@ -215,7 +215,7 @@ public class SnmpTransportContext extends TransportContext { return bindingsPerMethod.keySet().stream() .map(method -> { PDU request = new PDU(); - request.setType(getSnmpMethod(method)); + request.setType(method.getCode()); request.addAll(bindingsPerMethod.get(method)); return request; }) @@ -311,22 +311,9 @@ public class SnmpTransportContext extends TransportContext { return Optional.empty(); } - private Optional getMapping(OID responseOid, List mappings) { + private Optional getMapping(OID oid, List mappings) { return mappings.stream() - .filter(kvMapping -> new OID(kvMapping.getOid()).equals(responseOid)) - //TODO: OID shouldn't be duplicated in the config, add backend and UI verification + .filter(mapping -> new OID(mapping.getOid()).equals(oid)) .findFirst(); } - - private int getSnmpMethod(String configMethod) { - switch (configMethod) { - case "get": - return PDU.GET; - case "getNext": - case "response": - case "set": - default: - return -1; - } - } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 3e9eb63943..86d2cead95 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.Snmp; import org.snmp4j.event.ResponseEvent; +import org.snmp4j.smi.Null; import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultUdpTransportMapping; import org.springframework.context.annotation.Lazy; @@ -103,8 +104,8 @@ public class SnmpTransportService implements TbTransportService { try { log.debug("[{}] Sending SNMP message for device {}", pdu.getRequestID(), sessionContext.getDeviceId()); snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext); - } catch (IOException e) { - log.error(e.getMessage(), e); + } catch (Exception e) { + log.error("Failed to send SNMP request: {}", e.getMessage()); } }); } @@ -123,7 +124,7 @@ public class SnmpTransportService implements TbTransportService { PDU response = event.getResponse(); if (response == null) { - log.warn("No SNMP response, requestId: {}", event.getRequest().getRequestID()); + log.warn("No response from SNMP device {}, requestId: {}", sessionContext.getDeviceId(), event.getRequest().getRequestID()); return; } @@ -138,6 +139,11 @@ public class SnmpTransportService implements TbTransportService { VariableBinding variableBinding = response.get(i); log.trace("Processing variable binding {}: {}", i, variableBinding); + if (variableBinding.getVariable() instanceof Null) { + log.debug("Response variable is empty"); + continue; + } + snmpTransportContext.getTelemetryMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { log.trace("Found telemetry mapping for oid {}: {}", variableBinding.getOid(), mapping); processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), telemetry); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index ed87bbc726..3cfc9bc041 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -22,13 +22,12 @@ import org.snmp4j.CommunityTarget; import org.snmp4j.Target; import org.snmp4j.event.ResponseEvent; import org.snmp4j.event.ResponseListener; -import org.snmp4j.mp.SnmpConstants; import org.snmp4j.smi.GenericAddress; import org.snmp4j.smi.OctetString; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.SnmpProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; @@ -51,7 +50,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S private final String token; @Getter @Setter - private SnmpProfileTransportConfiguration profileTransportConfiguration; + private SnmpDeviceProfileTransportConfiguration profileTransportConfiguration; @Getter @Setter private SnmpDeviceTransportConfiguration deviceTransportConfiguration; @@ -79,7 +78,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S this.snmpTransportContext = snmpTransportContext; this.snmpTransportService = snmpTransportService; - this.profileTransportConfiguration = (SnmpProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + this.profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); this.deviceTransportConfiguration = deviceTransportConfiguration; initTarget(this.profileTransportConfiguration, this.deviceTransportConfiguration); @@ -105,11 +104,11 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S } } - public void initTarget(SnmpProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + public void initTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { log.trace("Initializing target for SNMP session of device {}", device); CommunityTarget communityTarget = new CommunityTarget(); communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); - communityTarget.setVersion(getSnmpVersion(deviceTransportConfig.getProtocolVersion())); + communityTarget.setVersion(deviceTransportConfig.getProtocolVersion().getCode()); communityTarget.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); communityTarget.setRetries(profileTransportConfig.getRetries()); @@ -125,20 +124,6 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S return token; } - //TODO: replace with enum, wtih preliminary discussion of type version in config (string or integer) - private int getSnmpVersion(String configSnmpVersion) { - switch (configSnmpVersion) { - case ("v1"): - return SnmpConstants.version1; - case ("v2c"): - return SnmpConstants.version2c; - case ("v3"): - return SnmpConstants.version3; - default: - return -1; - } - } - @Override public int nextMsgId() { return msgIdSeq.incrementAndGet(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 95f52cf55e..fea97bc903 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -350,6 +350,7 @@ public class DeviceProfileServiceImpl extends AbstractEntityService implements D } DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + transportConfiguration.validate(); if (transportConfiguration instanceof MqttDeviceProfileTransportConfiguration) { MqttDeviceProfileTransportConfiguration mqttTransportConfiguration = (MqttDeviceProfileTransportConfiguration) transportConfiguration; if (mqttTransportConfiguration.getTransportPayloadTypeConfiguration() instanceof ProtoTransportPayloadConfiguration) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 64b44db686..f3644d80ac 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.device.credentials.BasicMqttCredential import org.thingsboard.server.common.data.device.data.DefaultDeviceConfiguration; import org.thingsboard.server.common.data.device.data.DefaultDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.DeviceData; +import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.Lwm2mDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.MqttDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; @@ -604,6 +605,9 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe throw new DataValidationException("Can't assign device to customer from different tenant!"); } } + Optional.ofNullable(device.getDeviceData()) + .flatMap(deviceData -> Optional.ofNullable(deviceData.getTransportConfiguration())) + .ifPresent(DeviceTransportConfiguration::validate); } }; From 306ace040cb94fdd777e392af98f772f31bb2ee5 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 25 Mar 2021 10:06:06 +0200 Subject: [PATCH 08/32] Refactor --- .../src/main/resources/thingsboard.yml | 5 + .../JsonBasedTransportConfiguration.java | 39 ---- common/transport/snmp/pom.xml | 6 + .../transport/snmp/SnmpDeviceSimulator.java | 81 --------- .../transport/snmp/SnmpTransportContext.java | 10 +- .../snmp/service/SnmpTransportService.java | 34 +++- .../snmp/session/DeviceSessionContext.java | 13 +- .../transport/snmp/SnmpDeviceSimulator.java | 168 ++++++++++++++++++ .../snmp-device-profile-transport-config.json | 27 +++ .../snmp-device-transport-config.json | 6 + .../main/resources/device-data-config.json | 9 - .../main/resources/device-profile-config.json | 21 --- .../src/main/resources/tb-snmp-transport.yml | 5 + 13 files changed, 257 insertions(+), 167 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java delete mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java create mode 100644 common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java create mode 100644 common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json create mode 100644 common/transport/snmp/src/test/resources/snmp-device-transport-config.json delete mode 100644 transport/snmp/src/main/resources/device-data-config.json delete mode 100644 transport/snmp/src/main/resources/device-profile-config.json diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 3f7ef449ed..2cedd30bc8 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -647,6 +647,11 @@ transport: redis.enabled: "${LWM2M_REDIS_ENABLED:false}" snmp: enabled: "${SNMP_ENABLED:true}" + response_processing: + # parallelism level for executor (workStealingPool) that is responsible for handling responses from SNMP devices + parallelism_level: "${SNMP_RESPONSE_PROCESSING_PARALLELISM_LEVEL:20}" + # to configure SNMP to work over UDP or TCP + underlying_protocol: "${SNMP_UNDERLYING_PROTOCOL:udp}" swagger: api_path_regex: "${SWAGGER_API_PATH_REGEX:/api.*}" diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java deleted file mode 100644 index c65e9dd972..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/JsonBasedTransportConfiguration.java +++ /dev/null @@ -1,39 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.device; - -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; -import com.fasterxml.jackson.annotation.JsonIgnore; - -import java.util.HashMap; -import java.util.Map; - -public abstract class JsonBasedTransportConfiguration { - - @JsonIgnore - private final Map properties = new HashMap<>(); - - @JsonAnyGetter - public Map properties() { - return this.properties; - } - - @JsonAnySetter - public void put(String name, Object value) { - this.properties.put(name, value); - } -} diff --git a/common/transport/snmp/pom.xml b/common/transport/snmp/pom.xml index ab83ab69a7..afa35b518c 100644 --- a/common/transport/snmp/pom.xml +++ b/common/transport/snmp/pom.xml @@ -58,5 +58,11 @@ org.snmp4j snmp4j + + org.snmp4j + snmp4j-agent + 3.3.6 + test + diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java deleted file mode 100644 index 7109ef4e3f..0000000000 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.transport.snmp; - -import org.snmp4j.CommunityTarget; -import org.snmp4j.PDU; -import org.snmp4j.Snmp; -import org.snmp4j.Target; -import org.snmp4j.event.ResponseEvent; -import org.snmp4j.mp.SnmpConstants; -import org.snmp4j.smi.GenericAddress; -import org.snmp4j.smi.OID; -import org.snmp4j.smi.OctetString; -import org.snmp4j.smi.VariableBinding; -import org.snmp4j.transport.DefaultUdpTransportMapping; -import org.snmp4j.transport.UdpTransportMapping; - -import java.io.IOException; - -/** - * For testing purposes. Will be removed when the time comes - */ -public class SnmpDeviceSimulator { - private final Target target; - private final OID oid = new OID(".1.3.6.1.2.1.1.1.0"); - private Snmp snmp; - - public SnmpDeviceSimulator(int port) { - String address = "udp:127.0.0.1/" + port; - - CommunityTarget target = new CommunityTarget(); - target.setCommunity(new OctetString("public")); - target.setAddress(GenericAddress.parse(address)); - target.setRetries(2); - target.setTimeout(1500); - target.setVersion(SnmpConstants.version2c); - - this.target = target; - } - - public static void main(String[] args) throws IOException { - SnmpDeviceSimulator deviceSimulator = new SnmpDeviceSimulator(161); - - deviceSimulator.start(); - String response = deviceSimulator.sendRequest(PDU.GET); - - System.out.println(response); - } - - public void start() throws IOException { - UdpTransportMapping transport = new DefaultUdpTransportMapping(); - transport.addTransportListener((sourceTransport, incomingAddress, wholeMessage, tmStateReference) -> { - System.out.println(); - }); - snmp = new Snmp(transport); - - transport.listen(); - } - - public String sendRequest(int pduType) throws IOException { - PDU pdu = new PDU(); - pdu.add(new VariableBinding(oid)); - pdu.setType(pduType); - - ResponseEvent responseEvent = snmp.send(pdu, target); - return responseEvent.getResponse().get(0).getVariable().toString(); - } -} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index beaaef829f..2c1896a505 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -20,6 +20,7 @@ import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.smi.OID; import org.snmp4j.smi.VariableBinding; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Device; @@ -79,6 +80,9 @@ public class SnmpTransportContext extends TransportContext { private final Map> profilesPdus = new ConcurrentHashMap<>(); private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); + @Value("${transport.snmp.underlying_protocol}") + private String snmpUnderlyingProtocol; + @AfterStartUp(order = 2) public void initDevicesSessions() { log.info("Initializing SNMP devices sessions"); @@ -124,9 +128,9 @@ public class SnmpTransportContext extends TransportContext { profilesPdus.computeIfAbsent(deviceProfileId, id -> createPdus(profileTransportConfiguration)); DeviceSessionContext deviceSessionContext = new DeviceSessionContext( - device, deviceProfile, - credentials.getCredentialsId(), deviceTransportConfiguration, - this, snmpTransportService + device, deviceProfile, credentials.getCredentialsId(), + deviceTransportConfiguration, this, + snmpTransportService, snmpUnderlyingProtocol ); registerSessionMsgListener(deviceSessionContext); sessions.put(device.getId(), deviceSessionContext); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 86d2cead95..fd386e612a 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -19,10 +19,13 @@ import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; import org.snmp4j.PDU; import org.snmp4j.Snmp; +import org.snmp4j.TransportMapping; import org.snmp4j.event.ResponseEvent; import org.snmp4j.smi.Null; import org.snmp4j.smi.VariableBinding; +import org.snmp4j.transport.DefaultTcpTransportMapping; import org.snmp4j.transport.DefaultUdpTransportMapping; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; @@ -57,6 +60,11 @@ public class SnmpTransportService implements TbTransportService { private ScheduledExecutorService pollingExecutor; private ExecutorService snmpResponseProcessingExecutor; + @Value("${transport.snmp.response_processing.parallelism_level}") + private Integer responseProcessingParallelismLevel; + @Value("${transport.snmp.underlying_protocol}") + private String snmpUnderlyingProtocol; + public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext, TransportService transportService) { this.snmpTransportContext = snmpTransportContext; @@ -68,8 +76,7 @@ public class SnmpTransportService implements TbTransportService { log.info("Initializing SNMP transport service"); pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); - //TODO: Set parallelism value in the config - snmpResponseProcessingExecutor = Executors.newWorkStealingPool(20); + snmpResponseProcessingExecutor = Executors.newWorkStealingPool(responseProcessingParallelismLevel); initializeSnmp(); @@ -77,19 +84,28 @@ public class SnmpTransportService implements TbTransportService { } private void initializeSnmp() throws IOException { - snmp = new Snmp(new DefaultUdpTransportMapping()); + TransportMapping transportMapping; + switch (snmpUnderlyingProtocol) { + case "udp": + transportMapping = new DefaultUdpTransportMapping(); + break; + case "tcp": + transportMapping = new DefaultTcpTransportMapping(); + break; + default: + throw new IllegalArgumentException("Underlying protocol " + snmpUnderlyingProtocol + " for SNMP is not supported"); + } + snmp = new Snmp(transportMapping); snmp.listen(); } @AfterStartUp(order = 10) public void startPolling() { log.info("Starting SNMP polling"); - //TODO: Get poll period from configuration; - int pollPeriodSeconds = 1; pollingExecutor.scheduleWithFixedDelay(() -> { snmpTransportContext.getSessions().forEach(this::executeSnmpRequest); - }, 0, pollPeriodSeconds, TimeUnit.SECONDS); + }, 0, 1, TimeUnit.SECONDS); } private void executeSnmpRequest(DeviceSessionContext sessionContext) { @@ -111,12 +127,12 @@ public class SnmpTransportService implements TbTransportService { } } - public void onNewDeviceResponse(ResponseEvent responseEvent, DeviceSessionContext sessionContext) { + public void onNewDeviceResponse(DeviceSessionContext sessionContext, ResponseEvent responseEvent) { ((Snmp) responseEvent.getSource()).cancel(responseEvent.getRequest(), sessionContext); - snmpResponseProcessingExecutor.submit(() -> processSnmpResponse(responseEvent, sessionContext)); + snmpResponseProcessingExecutor.submit(() -> processSnmpResponse(sessionContext, responseEvent)); } - private void processSnmpResponse(ResponseEvent event, DeviceSessionContext sessionContext) { + private void processSnmpResponse(DeviceSessionContext sessionContext, ResponseEvent event) { if (event.getError() != null) { log.warn("Response error: {}", event.getError().getMessage(), event.getError()); return; diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index 3cfc9bc041..57e89ad034 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -65,10 +65,12 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S private long previousRequestExecutedAt = 0; private final AtomicInteger msgIdSeq = new AtomicInteger(0); private boolean isActive = true; + private final String snmpUnderlyingProtocol; - public DeviceSessionContext(Device device, DeviceProfile deviceProfile, - String token, SnmpDeviceTransportConfiguration deviceTransportConfiguration, - SnmpTransportContext snmpTransportContext, SnmpTransportService snmpTransportService) { + public DeviceSessionContext(Device device, DeviceProfile deviceProfile, String token, + SnmpDeviceTransportConfiguration deviceTransportConfiguration, + SnmpTransportContext snmpTransportContext, SnmpTransportService snmpTransportService, + String snmpUnderlyingProtocol) { super(UUID.randomUUID()); super.setDeviceId(device.getId()); super.setDeviceProfile(deviceProfile); @@ -81,6 +83,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S this.profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); this.deviceTransportConfiguration = deviceTransportConfiguration; + this.snmpUnderlyingProtocol = snmpUnderlyingProtocol; initTarget(this.profileTransportConfiguration, this.deviceTransportConfiguration); } @@ -100,14 +103,14 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onResponse(ResponseEvent event) { if (isActive) { - snmpTransportService.onNewDeviceResponse(event, this); + snmpTransportService.onNewDeviceResponse(this, event); } } public void initTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { log.trace("Initializing target for SNMP session of device {}", device); CommunityTarget communityTarget = new CommunityTarget(); - communityTarget.setAddress(GenericAddress.parse(GenericAddress.TYPE_UDP + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); + communityTarget.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); communityTarget.setVersion(deviceTransportConfig.getProtocolVersion().getCode()); communityTarget.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java new file mode 100644 index 0000000000..29fa35dcab --- /dev/null +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java @@ -0,0 +1,168 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import org.snmp4j.CommunityTarget; +import org.snmp4j.Target; +import org.snmp4j.TransportMapping; +import org.snmp4j.agent.BaseAgent; +import org.snmp4j.agent.CommandProcessor; +import org.snmp4j.agent.DuplicateRegistrationException; +import org.snmp4j.agent.MOGroup; +import org.snmp4j.agent.ManagedObject; +import org.snmp4j.agent.mo.MOAccessImpl; +import org.snmp4j.agent.mo.MOScalar; +import org.snmp4j.agent.mo.snmp.RowStatus; +import org.snmp4j.agent.mo.snmp.SnmpCommunityMIB; +import org.snmp4j.agent.mo.snmp.SnmpNotificationMIB; +import org.snmp4j.agent.mo.snmp.SnmpTargetMIB; +import org.snmp4j.agent.mo.snmp.StorageType; +import org.snmp4j.agent.mo.snmp.VacmMIB; +import org.snmp4j.agent.security.MutableVACM; +import org.snmp4j.mp.MPv3; +import org.snmp4j.mp.SnmpConstants; +import org.snmp4j.security.SecurityLevel; +import org.snmp4j.security.SecurityModel; +import org.snmp4j.security.USM; +import org.snmp4j.smi.Address; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.Integer32; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.Variable; +import org.snmp4j.transport.TransportMappings; + +import java.io.File; +import java.io.IOException; +import java.util.Map; +import java.util.Scanner; + +public class SnmpDeviceSimulator extends BaseAgent { + + public static void main(String[] args) throws IOException { + SnmpDeviceSimulator device = new SnmpDeviceSimulator(1610); + + device.start(); + device.setUpMappings(Map.of( + ".1.3.6.1.2.1.1.1.50", "12", + ".1.3.6.1.2.1.2.1.52", "56", + ".1.3.6.1.2.1.3.1.54", "yes" + )); + + new Scanner(System.in).nextLine(); + } + + + private final Target target; + private final Address address; + + public SnmpDeviceSimulator(int port) { + super(new File("conf.agent"), new File("bootCounter.agent"), + new CommandProcessor(new OctetString(MPv3.createLocalEngineID()))); + + CommunityTarget target = new CommunityTarget(); + target.setCommunity(new OctetString("public")); + address = GenericAddress.parse("udp:0.0.0.0/" + port); + target.setAddress(address); + target.setRetries(2); + target.setTimeout(1500); + target.setVersion(SnmpConstants.version2c); + this.target = target; + } + + public void start() throws IOException { + init(); + addShutdownHook(); + getServer().addContext(new OctetString("public")); + finishInit(); + run(); + sendColdStartNotification(); + } + + public void setUpMappings(Map oidToResponseMappings) { + unregisterManagedObject(getSnmpv2MIB()); + oidToResponseMappings.forEach((oid, response) -> { + registerManagedObject(new MOScalar<>(new OID(oid), MOAccessImpl.ACCESS_READ_ONLY, new OctetString(response))); + }); + } + + public Target getTarget() { + return target; + } + + @Override + protected void registerManagedObjects() { + } + + protected void registerManagedObject(ManagedObject mo) { + try { + server.register(mo, null); + } catch (DuplicateRegistrationException ex) { + throw new RuntimeException(ex); + } + } + + protected void unregisterManagedObject(MOGroup moGroup) { + moGroup.unregisterMOs(server, getContext(moGroup)); + } + + @Override + protected void addNotificationTargets(SnmpTargetMIB targetMIB, + SnmpNotificationMIB notificationMIB) { + } + + @Override + protected void addViews(VacmMIB vacm) { + vacm.addGroup(SecurityModel.SECURITY_MODEL_SNMPv2c, new OctetString( + "cpublic"), new OctetString("v1v2group"), + StorageType.nonVolatile); + + vacm.addAccess(new OctetString("v1v2group"), new OctetString("public"), + SecurityModel.SECURITY_MODEL_ANY, SecurityLevel.NOAUTH_NOPRIV, + MutableVACM.VACM_MATCH_EXACT, new OctetString("fullReadView"), + new OctetString("fullWriteView"), new OctetString( + "fullNotifyView"), StorageType.nonVolatile); + + vacm.addViewTreeFamily(new OctetString("fullReadView"), new OID("1.3"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + } + + protected void addUsmUser(USM usm) { + } + + protected void initTransportMappings() { + transportMappings = new TransportMapping[]{TransportMappings.getInstance().createTransportMapping(address)}; + } + + protected void unregisterManagedObjects() { + } + + protected void addCommunities(SnmpCommunityMIB communityMIB) { + Variable[] com2sec = new Variable[]{ + new OctetString("public"), + new OctetString("cpublic"), + getAgent().getContextEngineID(), + new OctetString("public"), + new OctetString(), + new Integer32(StorageType.nonVolatile), + new Integer32(RowStatus.active) + }; + SnmpCommunityMIB.SnmpCommunityEntryRow row = communityMIB.getSnmpCommunityEntry().createRow( + new OctetString("public2public").toSubIndex(true), com2sec); + communityMIB.getSnmpCommunityEntry().addRow(row); + } +} diff --git a/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json b/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json new file mode 100644 index 0000000000..78b66c4c0e --- /dev/null +++ b/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json @@ -0,0 +1,27 @@ +{ + "pollPeriodMs": 3000, + "timeoutMs": 500, + "retries": 0, + "telemetryMappings": [ + { + "oid": ".1.3.6.1.2.1.1.1.50", + "method": "GET", + "key": "temperature", + "dataType": "LONG" + }, + { + "oid": ".1.3.6.1.2.1.2.1.52", + "method": "GET", + "key": "humidity", + "dataType": "LONG" + } + ], + "attributesMappings": [ + { + "oid": ".1.3.6.1.2.1.3.1.54", + "method": "GET", + "key": "isCool", + "dataType": "STRING" + } + ] +} \ No newline at end of file diff --git a/common/transport/snmp/src/test/resources/snmp-device-transport-config.json b/common/transport/snmp/src/test/resources/snmp-device-transport-config.json new file mode 100644 index 0000000000..c73d817bfb --- /dev/null +++ b/common/transport/snmp/src/test/resources/snmp-device-transport-config.json @@ -0,0 +1,6 @@ +{ + "address": "127.0.0.1", + "port": 1610, + "community": "public", + "protocolVersion": "V2C" +} \ No newline at end of file diff --git a/transport/snmp/src/main/resources/device-data-config.json b/transport/snmp/src/main/resources/device-data-config.json deleted file mode 100644 index e425ccc3a6..0000000000 --- a/transport/snmp/src/main/resources/device-data-config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "deviceName": "Thermostat T1", - "snmpConfig": { - "address": "192.168.1.2", - "port": 161, - "community": "U5J=$HWj6f@7", - "protocolVersion": "v2c" - } -} diff --git a/transport/snmp/src/main/resources/device-profile-config.json b/transport/snmp/src/main/resources/device-profile-config.json deleted file mode 100644 index b272913045..0000000000 --- a/transport/snmp/src/main/resources/device-profile-config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "poolPeriodMs": 10000, - "timeoutMs": 5000, - "retries": 5, - "attributes": [ - { - "key": "snmpNodeManagerEmail", - "type": "STRING", - "method": "get", - "oid": ".1.3.6.1.2.1.1.4.0" - } - ], - "telemetry": [ - { - "key": "snmpNodeSysUpTime", - "type": "LONG", - "method": "get", - "oid": ".1.3.6.1.2.1.1.3.0" - } - ] -} \ No newline at end of file diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 9f6bd131ac..db9f79b51a 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -189,6 +189,11 @@ transport: # Local LwM2M transport parameters snmp: enabled: "${SNMP_ENABLED:true}" + response_processing: + # parallelism level for executor (workStealingPool) that is responsible for handling responses from SNMP devices + parallelism_level: "${SNMP_RESPONSE_PROCESSING_PARALLELISM_LEVEL:20}" + # to configure SNMP to work over UDP or TCP + underlying_protocol: "${SNMP_UNDERLYING_PROTOCOL:udp}" queue: type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) From 61341a7cafdf9cbbe59fc8b31bc7827b1ad19382 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 1 Apr 2021 15:50:37 +0300 Subject: [PATCH 09/32] Refactor, implement SNMP traps, add support for SNMP v3 targets --- .../SnmpDeviceTransportConfiguration.java | 6 +- ...mpDeviceProfileTransportConfiguration.java | 23 +- .../transport/snmp/SnmpCommunicationSpec.java | 40 +++ .../data/transport/snmp/SnmpMapping.java | 3 +- ...ibutesQueryingSnmpCommunicationConfig.java | 25 ++ ...TrapsReceivingSnmpCommunicationConfig.java | 25 ++ ...eatingQueryingSnmpCommunicationConfig.java | 36 +++ ...ributesSettingSnmpCommunicationConfig.java | 36 +++ .../snmp/configs/SnmpCommunicationConfig.java | 62 +++++ ...emetryQueryingSnmpCommunicationConfig.java | 29 ++ ...TrapsReceivingSnmpCommunicationConfig.java | 30 ++ .../transport/snmp/SnmpTransportContext.java | 83 +----- .../snmp/service/SnmpTransportService.java | 263 ++++++++++++++---- .../snmp/session/DeviceSessionContext.java | 105 ++++++- ...ulator.java => SnmpDeviceSimulatorV2.java} | 76 ++++- .../transport/snmp/SnmpDeviceSimulatorV3.java | 52 ++++ .../server/transport/snmp/SnmpTest.java | 49 ++++ .../snmp-device-profile-transport-config.json | 62 +++-- .../snmp-device-transport-config.json | 2 +- .../transport/adaptor/JsonConverter.java | 2 +- 20 files changed, 810 insertions(+), 199 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesQueryingSnmpCommunicationConfig.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryQueryingSnmpCommunicationConfig.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java rename common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/{SnmpDeviceSimulator.java => SnmpDeviceSimulatorV2.java} (69%) create mode 100644 common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java create mode 100644 common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTest.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index 551bf2fcfb..a6056b4188 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -25,8 +25,10 @@ import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; public class SnmpDeviceTransportConfiguration implements DeviceTransportConfiguration { private String address; private int port; - private String community; private SnmpProtocolVersion protocolVersion; + private String securityName; + private String authenticationPassphrase; // for SNMP v3 + private String privacyPassphrase; // for SNMP v3 @Override public DeviceTransportType getType() { @@ -43,6 +45,6 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur @JsonIgnore private boolean isValid() { return StringUtils.isNotBlank(address) && port > 0 && - StringUtils.isNotBlank(community) && protocolVersion != null; + StringUtils.isNotBlank(securityName) && protocolVersion != null; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java index a4f7cc57de..843a51304e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java @@ -18,35 +18,21 @@ package org.thingsboard.server.common.data.device.profile; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.DeviceTransportType; -import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.configs.SnmpCommunicationConfig; -import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; @Data public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTransportConfiguration { - private int pollPeriodMs; private int timeoutMs; private int retries; - private List telemetryMappings; - private List attributesMappings; + private List communicationConfigs; @Override public DeviceTransportType getType() { return DeviceTransportType.SNMP; } - @JsonIgnore - public List getAllMappings() { - if (telemetryMappings != null && attributesMappings != null) { - return Stream.concat(telemetryMappings.stream(), attributesMappings.stream()).collect(Collectors.toList()); - } else { - return Collections.emptyList(); - } - } - @Override public void validate() { if (!isValid()) { @@ -56,9 +42,6 @@ public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTra @JsonIgnore private boolean isValid() { - List mappings = getAllMappings(); - return pollPeriodMs > 0 && timeoutMs > 0 && retries >= 0 && - !mappings.isEmpty() && mappings.stream().allMatch(SnmpMapping::isValid) && - mappings.stream().map(SnmpMapping::getOid).distinct().count() == mappings.size(); + return true; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java new file mode 100644 index 0000000000..f6ff7d3109 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp; + +public enum SnmpCommunicationSpec { + TELEMETRY_QUERYING(true), + CLIENT_ATTRIBUTES_QUERYING(true), + + SHARED_ATTRIBUTES_SETTING, + + TELEMETRY_TRAPS_RECEIVING, + CLIENT_ATTRIBUTES_TRAPS_RECEIVING; + + private final boolean isRepeatingQuerying; + + SnmpCommunicationSpec() { + this.isRepeatingQuerying = false; + } + + SnmpCommunicationSpec(boolean isRepeatingQuerying) { + this.isRepeatingQuerying = isRepeatingQuerying; + } + + public boolean isRepeatingQuerying() { + return isRepeatingQuerying; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java index 32a3f2a25b..0269491461 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java @@ -25,7 +25,6 @@ import java.util.regex.Pattern; @Data public class SnmpMapping { private String oid; - private SnmpMethod method; private String key; private DataType dataType; @@ -33,7 +32,7 @@ public class SnmpMapping { @JsonIgnore public boolean isValid() { - return StringUtils.isNotEmpty(oid) && OID_PATTERN.matcher(oid).matches() && method != null && + return StringUtils.isNotEmpty(oid) && OID_PATTERN.matcher(oid).matches() && StringUtils.isNotBlank(key) && dataType != null; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesQueryingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..16198f8ef6 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesQueryingSnmpCommunicationConfig.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.configs; + +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; + +public class ClientAttributesQueryingSnmpCommunicationConfig extends RepeatingQueryingSnmpCommunicationConfig { + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..9a11b01f1b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java @@ -0,0 +1,25 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.configs; + +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; + +public class ClientAttributesTrapsReceivingSnmpCommunicationConfig extends SnmpCommunicationConfig { + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.CLIENT_ATTRIBUTES_TRAPS_RECEIVING; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..764bf97b15 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.configs; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; + +@EqualsAndHashCode(callSuper = true) +@Data +public abstract class RepeatingQueryingSnmpCommunicationConfig extends SnmpCommunicationConfig { + private Long queryingFrequencyMs; + + @Override + public SnmpMethod getMethod() { + return SnmpMethod.GET; + } + + @Override + public boolean isValid() { + return true; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..1c63dae618 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.configs; + +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; + +public class SharedAttributesSettingSnmpCommunicationConfig extends SnmpCommunicationConfig { + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.SHARED_ATTRIBUTES_SETTING; + } + + @Override + public SnmpMethod getMethod() { + return SnmpMethod.SET; + } + + @Override + public boolean isValid() { + return true; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java new file mode 100644 index 0000000000..bfc194669b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java @@ -0,0 +1,62 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.configs; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonSubTypes.Type; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; + +import java.util.List; + +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "spec") +@JsonSubTypes({ + @Type(value = TelemetryQueryingSnmpCommunicationConfig.class, name = "TELEMETRY_QUERYING"), + @Type(value = ClientAttributesQueryingSnmpCommunicationConfig.class, name = "CLIENT_ATTRIBUTES_QUERYING"), + @Type(value = SharedAttributesSettingSnmpCommunicationConfig.class, name = "SHARED_ATTRIBUTES_SETTING"), + @Type(value = TelemetryTrapsReceivingSnmpCommunicationConfig.class, name = "TELEMETRY_TRAPS_RECEIVING"), + @Type(value = ClientAttributesTrapsReceivingSnmpCommunicationConfig.class, name = "CLIENT_ATTRIBUTES_TRAPS_RECEIVING") +}) +public abstract class SnmpCommunicationConfig { + protected List mappings; + + public List getMappings() { + return mappings; + } + + public abstract SnmpCommunicationSpec getSpec(); + + @JsonIgnore + public SnmpMethod getMethod() { + return null; + } + + @JsonIgnore + public boolean isValid() { + return true; + } + + public void validate() { + if (!isValid()) { + throw new IllegalArgumentException("Communication config is not valid"); + } + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryQueryingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..68d3c81ed2 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryQueryingSnmpCommunicationConfig.java @@ -0,0 +1,29 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.configs; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; + +@EqualsAndHashCode(callSuper = true) +@Data +public class TelemetryQueryingSnmpCommunicationConfig extends RepeatingQueryingSnmpCommunicationConfig { + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.TELEMETRY_QUERYING; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..66be4d0cf8 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.configs; + +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; + +public class TelemetryTrapsReceivingSnmpCommunicationConfig extends SnmpCommunicationConfig { + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.TELEMETRY_TRAPS_RECEIVING; + } + + @Override + public boolean isValid() { + return false; + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 2c1896a505..877d4f1652 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -17,9 +17,6 @@ package org.thingsboard.server.transport.snmp; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.snmp4j.PDU; -import org.snmp4j.smi.OID; -import org.snmp4j.smi.VariableBinding; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; @@ -33,8 +30,6 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; -import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; -import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; import org.thingsboard.server.common.transport.DeviceUpdatedEvent; import org.thingsboard.server.common.transport.TransportContext; import org.thingsboard.server.common.transport.TransportDeviceProfileCache; @@ -42,7 +37,6 @@ import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; -import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.queue.util.AfterStartUp; @@ -52,9 +46,7 @@ import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingServi import org.thingsboard.server.transport.snmp.service.SnmpTransportService; import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; -import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -76,8 +68,6 @@ public class SnmpTransportContext extends TransportContext { private final SnmpTransportBalancingService balancingService; private final Map sessions = new ConcurrentHashMap<>(); - private final Map profilesTransportConfigs = new ConcurrentHashMap<>(); - private final Map> profilesPdus = new ConcurrentHashMap<>(); private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); @Value("${transport.snmp.underlying_protocol}") @@ -124,22 +114,19 @@ public class SnmpTransportContext extends TransportContext { SnmpDeviceProfileTransportConfiguration profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - profilesTransportConfigs.put(deviceProfileId, profileTransportConfiguration); - profilesPdus.computeIfAbsent(deviceProfileId, id -> createPdus(profileTransportConfiguration)); - DeviceSessionContext deviceSessionContext = new DeviceSessionContext( device, deviceProfile, credentials.getCredentialsId(), - deviceTransportConfiguration, this, - snmpTransportService, snmpUnderlyingProtocol + profileTransportConfiguration, deviceTransportConfiguration, + this, snmpTransportService, snmpUnderlyingProtocol ); registerSessionMsgListener(deviceSessionContext); sessions.put(device.getId(), deviceSessionContext); + snmpTransportService.createQueryingTasks(deviceSessionContext); log.info("Established SNMP device session for device {}", device.getId()); } private void updateDeviceSession(DeviceSessionContext sessionContext, Device device, DeviceProfile deviceProfile) { log.info("Updating SNMP device session for device {}", device.getId()); - DeviceProfileId deviceProfileId = deviceProfile.getId(); DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId()); if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { @@ -152,14 +139,13 @@ public class SnmpTransportContext extends TransportContext { SnmpDeviceTransportConfiguration newDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); if (!newProfileTransportConfiguration.equals(sessionContext.getProfileTransportConfiguration())) { - profilesPdus.put(deviceProfileId, createPdus(newProfileTransportConfiguration)); - profilesTransportConfigs.put(deviceProfileId, newProfileTransportConfiguration); - sessionContext.setProfileTransportConfiguration(newProfileTransportConfiguration); - sessionContext.initTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); + sessionContext.initializeTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); + snmpTransportService.cancelQueryingTasks(sessionContext); + snmpTransportService.createQueryingTasks(sessionContext); } else if (!newDeviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { sessionContext.setDeviceTransportConfiguration(newDeviceTransportConfiguration); - sessionContext.initTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); + sessionContext.initializeTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); } else { log.trace("Configuration of the device {} was not updated", device); } @@ -171,22 +157,14 @@ public class SnmpTransportContext extends TransportContext { sessionContext.close(); transportService.deregisterSession(sessionContext.getSessionInfo()); sessions.remove(sessionContext.getDeviceId()); - log.trace("Deregistered and removed session"); - - DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); - if (sessions.values().stream() - .map(DeviceAwareSessionContext::getDeviceProfile) - .noneMatch(deviceProfile -> deviceProfile.getId().equals(deviceProfileId))) { - log.trace("Removed values for device profile {} from configs and pdus caches", deviceProfileId); - profilesTransportConfigs.remove(deviceProfileId); - profilesPdus.remove(deviceProfileId); - } + snmpTransportService.cancelQueryingTasks(sessionContext); + log.trace("Unregistered and removed session"); } private void registerSessionMsgListener(DeviceSessionContext deviceSessionContext) { transportService.process(DeviceTransportType.SNMP, TransportProtos.ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceSessionContext.getToken()).build(), - new TransportServiceCallback() { + new TransportServiceCallback<>() { @Override public void onSuccess(ValidateDeviceCredentialsResponse msg) { if (msg.hasDeviceInfo()) { @@ -195,6 +173,7 @@ public class SnmpTransportContext extends TransportContext { ); transportService.registerAsyncSession(sessionInfo, deviceSessionContext); + transportService.process(sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().build(), null); deviceSessionContext.setSessionInfo(sessionInfo); deviceSessionContext.setDeviceInfo(msg.getDeviceInfo()); } else { @@ -209,23 +188,6 @@ public class SnmpTransportContext extends TransportContext { }); } - private List createPdus(SnmpDeviceProfileTransportConfiguration deviceProfileConfig) { - Map> bindingsPerMethod = new HashMap<>(); - - deviceProfileConfig.getAllMappings().forEach(mapping -> bindingsPerMethod - .computeIfAbsent(mapping.getMethod(), v -> new ArrayList<>()) - .add(new VariableBinding(new OID(mapping.getOid())))); - - return bindingsPerMethod.keySet().stream() - .map(method -> { - PDU request = new PDU(); - request.setType(method.getCode()); - request.addAll(bindingsPerMethod.get(method)); - return request; - }) - .collect(Collectors.toList()); - } - @EventListener(DeviceUpdatedEvent.class) public void onDeviceUpdatedOrCreated(DeviceUpdatedEvent deviceUpdatedEvent) { Device device = deviceUpdatedEvent.getDevice(); @@ -297,27 +259,4 @@ public class SnmpTransportContext extends TransportContext { return sessions.values(); } - public Map> getProfilesPdus() { - return profilesPdus; - } - - public Optional getAttributeMapping(DeviceProfileId deviceProfileId, OID responseOid) { - if (profilesTransportConfigs.containsKey(deviceProfileId)) { - return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getAttributesMappings()); - } - return Optional.empty(); - } - - public Optional getTelemetryMapping(DeviceProfileId deviceProfileId, OID responseOid) { - if (profilesTransportConfigs.containsKey(deviceProfileId)) { - return getMapping(responseOid, profilesTransportConfigs.get(deviceProfileId).getTelemetryMappings()); - } - return Optional.empty(); - } - - private Optional getMapping(OID oid, List mappings) { - return mappings.stream() - .filter(mapping -> new OID(mapping.getOid()).equals(oid)) - .findFirst(); - } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index fd386e612a..963b5db382 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -16,12 +16,20 @@ package org.thingsboard.server.transport.snmp.service; import com.google.gson.JsonObject; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.snmp4j.CommandResponder; +import org.snmp4j.CommandResponderEvent; import org.snmp4j.PDU; import org.snmp4j.Snmp; import org.snmp4j.TransportMapping; import org.snmp4j.event.ResponseEvent; import org.snmp4j.smi.Null; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.TcpAddress; +import org.snmp4j.smi.UdpAddress; import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultTcpTransportMapping; import org.snmp4j.transport.DefaultUdpTransportMapping; @@ -32,11 +40,14 @@ import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.configs.RepeatingQueryingSnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.configs.SnmpCommunicationConfig; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; import org.thingsboard.server.transport.snmp.SnmpTransportContext; import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; @@ -44,21 +55,33 @@ import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.stream.Collectors; +import java.util.stream.Stream; @TbSnmpTransportComponent @Service @Slf4j -public class SnmpTransportService implements TbTransportService { +public class SnmpTransportService implements TbTransportService, CommandResponder { private final SnmpTransportContext snmpTransportContext; private final TransportService transportService; + @Getter private Snmp snmp; - private ScheduledExecutorService pollingExecutor; - private ExecutorService snmpResponseProcessingExecutor; + private ScheduledExecutorService queryingExecutor; + private ExecutorService responseProcessingExecutor; + + private final Map> responseProcessors = new EnumMap<>(SnmpCommunicationSpec.class); @Value("${transport.snmp.response_processing.parallelism_level}") private Integer responseProcessingParallelismLevel; @@ -75,10 +98,12 @@ public class SnmpTransportService implements TbTransportService { private void init() throws IOException { log.info("Initializing SNMP transport service"); - pollingExecutor = Executors.newScheduledThreadPool(1, ThingsBoardThreadFactory.forName("snmp-polling")); - snmpResponseProcessingExecutor = Executors.newWorkStealingPool(responseProcessingParallelismLevel); + queryingExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("snmp-querying")); + responseProcessingExecutor = Executors.newWorkStealingPool(responseProcessingParallelismLevel); initializeSnmp(); + initializeTrapsListener(); + configureResponseProcessors(); log.info("SNMP transport service initialized"); } @@ -99,40 +124,118 @@ public class SnmpTransportService implements TbTransportService { snmp.listen(); } - @AfterStartUp(order = 10) - public void startPolling() { - log.info("Starting SNMP polling"); + private void initializeTrapsListener() throws IOException { + int trapsListeningPort = 1062; + String bindingAddress = "0.0.0.0/" + trapsListeningPort; - pollingExecutor.scheduleWithFixedDelay(() -> { - snmpTransportContext.getSessions().forEach(this::executeSnmpRequest); - }, 0, 1, TimeUnit.SECONDS); - } + TransportMapping transportMapping; + switch (snmpUnderlyingProtocol) { + case "udp": + transportMapping = new DefaultUdpTransportMapping(new UdpAddress(bindingAddress)); + break; + case "tcp": + transportMapping = new DefaultTcpTransportMapping(new TcpAddress(bindingAddress)); + break; + default: + throw new IllegalArgumentException("Underlying protocol " + snmpUnderlyingProtocol + " for SNMP is not supported"); + } + + + Snmp trapsSnmp = new Snmp(transportMapping); + trapsSnmp.addCommandResponder(this); - private void executeSnmpRequest(DeviceSessionContext sessionContext) { - long timeNow = System.currentTimeMillis(); - long nextRequestExecutionTime = sessionContext.getPreviousRequestExecutedAt() + sessionContext.getProfileTransportConfiguration().getPollPeriodMs(); + transportMapping.listen(); + } - if (nextRequestExecutionTime < timeNow) { - sessionContext.setPreviousRequestExecutedAt(timeNow); + public void createQueryingTasks(DeviceSessionContext sessionContext) { + List> queryingTasks = sessionContext.getProfileTransportConfiguration().getCommunicationConfigs().stream() + .filter(config -> config.getSpec().isRepeatingQuerying()) + .map(config -> { + RepeatingQueryingSnmpCommunicationConfig repeatingCommunicationConfig = (RepeatingQueryingSnmpCommunicationConfig) config; + return createQueryingTaskForConfig(sessionContext, repeatingCommunicationConfig); + }) + .collect(Collectors.toList()); + sessionContext.setQueryingTasks(queryingTasks); + } - DeviceProfileId deviceProfileId = sessionContext.getDeviceProfile().getId(); - snmpTransportContext.getProfilesPdus().get(deviceProfileId).forEach(pdu -> { - try { - log.debug("[{}] Sending SNMP message for device {}", pdu.getRequestID(), sessionContext.getDeviceId()); - snmp.send(pdu, sessionContext.getTarget(), deviceProfileId, sessionContext); - } catch (Exception e) { - log.error("Failed to send SNMP request: {}", e.getMessage()); + private ScheduledFuture createQueryingTaskForConfig(DeviceSessionContext sessionContext, RepeatingQueryingSnmpCommunicationConfig communicationConfig) { + Long queryingFrequency = communicationConfig.getQueryingFrequencyMs(); + return queryingExecutor.scheduleWithFixedDelay(() -> { + try { + if (sessionContext.isActive()) { + sendRequest(sessionContext, communicationConfig); } - }); + } catch (Exception e) { + log.error("Failed to send SNMP request for device {}: {}", sessionContext.getDeviceId(), e.getMessage()); + } + }, queryingFrequency, queryingFrequency, TimeUnit.MILLISECONDS); + } + + public void cancelQueryingTasks(DeviceSessionContext sessionContext) { + sessionContext.getQueryingTasks().forEach(task -> task.cancel(true)); + sessionContext.getQueryingTasks().clear(); + } + + public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) throws IOException { + PDU request = createPdu(communicationConfig); + executeRequest(sessionContext, request); + } + + public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) throws IOException { + PDU request = createPduWithValues(communicationConfig, values); + executeRequest(sessionContext, request); + } + + private void executeRequest(DeviceSessionContext sessionContext, PDU request) throws IOException { + if (request.size() > 0) { + log.trace("Executing SNMP request for device {}. Variables bindings: {}", sessionContext.getDeviceId(), request.getVariableBindings()); + snmp.send(request, sessionContext.getTarget(), sessionContext.getDeviceProfile().getId(), sessionContext); } } - public void onNewDeviceResponse(DeviceSessionContext sessionContext, ResponseEvent responseEvent) { - ((Snmp) responseEvent.getSource()).cancel(responseEvent.getRequest(), sessionContext); - snmpResponseProcessingExecutor.submit(() -> processSnmpResponse(sessionContext, responseEvent)); + private PDU createPdu(SnmpCommunicationConfig communicationConfig) { + PDU pdu = new PDU(); + pdu.setType(communicationConfig.getMethod().getCode()); + pdu.addAll(communicationConfig.getMappings().stream() + .map(mapping -> new VariableBinding(new OID(mapping.getOid()))) + .collect(Collectors.toList())); + return pdu; } - private void processSnmpResponse(DeviceSessionContext sessionContext, ResponseEvent event) { + private PDU createPduWithValues(SnmpCommunicationConfig communicationConfig, Map values) { + PDU pdu = new PDU(); + pdu.setType(communicationConfig.getMethod().getCode()); + pdu.addAll(communicationConfig.getMappings().stream() + .filter(mapping -> values.containsKey(mapping.getKey())) + .map(mapping -> { + String value = values.get(mapping.getKey()); + return new VariableBinding(new OID(mapping.getOid()), new OctetString(value)); + }) + .collect(Collectors.toList())); + return pdu; + } + + + private void processTrap(CommandResponderEvent event) { + if (event.getPDU().getType() != PDU.TRAP) return; + + snmpTransportContext.getSessions().stream() + .filter(sessionContext -> { + // TODO: SNMP v3 support + return sessionContext.getTarget().getSecurityName().equals(OctetString.fromByteArray(event.getSecurityName())) && + sessionContext.getTarget().getAddress().equals(event.getPeerAddress()); + }) + .findFirst() + .ifPresentOrElse(sessionContext -> { + responseProcessingExecutor.execute(() -> processResponse(sessionContext, event.getPDU())); + }, () -> { + log.debug("SNMP event is from unknown source: {}", event); + }); + } + + public void processResponseEvent(DeviceSessionContext sessionContext, ResponseEvent event) { + ((Snmp) event.getSource()).cancel(event.getRequest(), sessionContext); + if (event.getError() != null) { log.warn("Response error: {}", event.getError().getMessage(), event.getError()); return; @@ -143,16 +246,29 @@ public class SnmpTransportService implements TbTransportService { log.warn("No response from SNMP device {}, requestId: {}", sessionContext.getDeviceId(), event.getRequest().getRequestID()); return; } - DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); log.debug("[{}] Processing SNMP response for device {} with device profile {}: {}", response.getRequestID(), sessionContext.getDeviceId(), deviceProfileId, response); - JsonObject telemetry = new JsonObject(); - JsonObject attributes = new JsonObject(); + responseProcessingExecutor.execute(() -> processResponse(sessionContext, response)); + } - for (int i = 0; i < response.size(); i++) { - VariableBinding variableBinding = response.get(i); + private void processResponse(DeviceSessionContext sessionContext, PDU responsePdu) { + Map mappings = new HashMap<>(); + Map configs = new HashMap<>(); + Map responses = new EnumMap<>(SnmpCommunicationSpec.class); + + for (SnmpCommunicationConfig config : sessionContext.getProfileTransportConfiguration().getCommunicationConfigs()) { + for (SnmpMapping mapping : config.getMappings()) { + OID oid = new OID(mapping.getOid()); + mappings.put(oid, mapping); + configs.put(oid, config); + } + responses.put(config.getSpec(), new JsonObject()); + } + + for (int i = 0; i < responsePdu.size(); i++) { + VariableBinding variableBinding = responsePdu.get(i); log.trace("Processing variable binding {}: {}", i, variableBinding); if (variableBinding.getVariable() instanceof Null) { @@ -160,37 +276,57 @@ public class SnmpTransportService implements TbTransportService { continue; } - snmpTransportContext.getTelemetryMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { - log.trace("Found telemetry mapping for oid {}: {}", variableBinding.getOid(), mapping); - processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), telemetry); - }); + OID oid = variableBinding.getOid(); + if (!mappings.containsKey(oid)) { + log.debug("No SNMP mapping for oid {}", oid); + continue; + } - snmpTransportContext.getAttributeMapping(deviceProfileId, variableBinding.getOid()).ifPresent(mapping -> { - log.trace("Found attribute mapping for oid {}: {}", variableBinding.getOid(), mapping); - processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), attributes); - }); - } + SnmpCommunicationSpec spec = configs.get(oid).getSpec(); + if (!responseProcessors.containsKey(spec)) { + log.debug("No response processor found for spec {}", spec); + continue; + } - if (telemetry.entrySet().isEmpty() && attributes.entrySet().isEmpty()) { - log.warn("No telemetry or attribute values is the SNMP response for device {}", sessionContext.getDeviceId()); - return; + SnmpMapping mapping = mappings.get(oid); + processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), responses.get(spec)); } - if (!telemetry.entrySet().isEmpty()) { - TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(telemetry); - transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, TransportServiceCallback.EMPTY); - log.debug("Posted telemetry for device {}: {}", sessionContext.getDeviceId(), telemetry); + if (responses.values().stream().allMatch(response -> response.entrySet().isEmpty())) { + log.debug("No values is the SNMP response for device {}. Request id: {}", sessionContext.getDeviceId(), responsePdu.getRequestID()); + return; } - if (!attributes.entrySet().isEmpty()) { - TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(attributes); - transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, TransportServiceCallback.EMPTY); - log.debug("Posted attributes for device {}: {}", sessionContext.getDeviceId(), attributes); - } + responses.forEach((spec, response) -> { + Optional.ofNullable(responseProcessors.get(spec)) + .ifPresent(responseProcessor -> { + responseProcessor.accept(response, sessionContext); + }); + }); reportActivity(sessionContext.getSessionInfo()); } + private void configureResponseProcessors() { + Stream.of(SnmpCommunicationSpec.TELEMETRY_QUERYING, SnmpCommunicationSpec.TELEMETRY_TRAPS_RECEIVING) + .forEach(telemetrySpec -> { + responseProcessors.put(telemetrySpec, (response, sessionContext) -> { + TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(response); + transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, TransportServiceCallback.EMPTY); + log.debug("Posted telemetry for device {}: {}", sessionContext.getDeviceId(), response); + }); + }); + + Stream.of(SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING, SnmpCommunicationSpec.CLIENT_ATTRIBUTES_TRAPS_RECEIVING) + .forEach(clientAttributesSpec -> { + responseProcessors.put(clientAttributesSpec, (response, sessionContext) -> { + TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(response); + transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, TransportServiceCallback.EMPTY); + log.debug("Posted attributes for device {}: {}", sessionContext.getDeviceId(), response); + }); + }); + } + private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { transportService.process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() .setAttributeSubscription(false) @@ -200,6 +336,8 @@ public class SnmpTransportService implements TbTransportService { } private void processValue(String key, DataType dataType, String value, JsonObject result) { + if (StringUtils.isEmpty(value)) return; + switch (dataType) { case LONG: result.addProperty(key, Long.parseLong(value)); @@ -215,6 +353,11 @@ public class SnmpTransportService implements TbTransportService { } } + @Override + public void processPdu(CommandResponderEvent event) { + processTrap(event); + } + @Override public String getName() { return "SNMP"; @@ -223,11 +366,11 @@ public class SnmpTransportService implements TbTransportService { @PreDestroy public void shutdown() { log.info("Stopping SNMP transport!"); - if (pollingExecutor != null) { - pollingExecutor.shutdownNow(); + if (queryingExecutor != null) { + queryingExecutor.shutdownNow(); } - if (snmpResponseProcessingExecutor != null) { - snmpResponseProcessingExecutor.shutdownNow(); + if (responseProcessingExecutor != null) { + responseProcessingExecutor.shutdownNow(); } if (snmp != null) { try { diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index 57e89ad034..c0bbc843bc 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -18,18 +18,31 @@ package org.thingsboard.server.transport.snmp.session; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.snmp4j.AbstractTarget; import org.snmp4j.CommunityTarget; import org.snmp4j.Target; +import org.snmp4j.UserTarget; import org.snmp4j.event.ResponseEvent; import org.snmp4j.event.ResponseListener; +import org.snmp4j.security.AuthSHA; +import org.snmp4j.security.PrivDES; +import org.snmp4j.security.SecurityLevel; +import org.snmp4j.security.SecurityModel; +import org.snmp4j.security.SecurityModels; +import org.snmp4j.security.USM; +import org.snmp4j.security.UsmUser; import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; import org.thingsboard.server.common.transport.SessionMsgListener; +import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; @@ -40,8 +53,13 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseM import org.thingsboard.server.transport.snmp.SnmpTransportContext; import org.thingsboard.server.transport.snmp.service.SnmpTransportService; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.UUID; +import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; @Slf4j public class DeviceSessionContext extends DeviceAwareSessionContext implements SessionMsgListener, ResponseListener { @@ -64,10 +82,16 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Setter private long previousRequestExecutedAt = 0; private final AtomicInteger msgIdSeq = new AtomicInteger(0); + @Getter private boolean isActive = true; private final String snmpUnderlyingProtocol; + @Getter + @Setter + private List> queryingTasks = new LinkedList<>(); + public DeviceSessionContext(Device device, DeviceProfile deviceProfile, String token, + SnmpDeviceProfileTransportConfiguration profileTransportConfiguration, SnmpDeviceTransportConfiguration deviceTransportConfiguration, SnmpTransportContext snmpTransportContext, SnmpTransportService snmpTransportService, String snmpUnderlyingProtocol) { @@ -80,11 +104,11 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S this.snmpTransportContext = snmpTransportContext; this.snmpTransportService = snmpTransportService; - this.profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); + this.profileTransportConfiguration = profileTransportConfiguration; this.deviceTransportConfiguration = deviceTransportConfiguration; this.snmpUnderlyingProtocol = snmpUnderlyingProtocol; - initTarget(this.profileTransportConfiguration, this.deviceTransportConfiguration); + initializeTarget(profileTransportConfiguration, deviceTransportConfiguration); } @Override @@ -103,20 +127,62 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onResponse(ResponseEvent event) { if (isActive) { - snmpTransportService.onNewDeviceResponse(this, event); + snmpTransportService.processResponseEvent(this, event); } } - public void initTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + public void initializeTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { log.trace("Initializing target for SNMP session of device {}", device); - CommunityTarget communityTarget = new CommunityTarget(); - communityTarget.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); - communityTarget.setVersion(deviceTransportConfig.getProtocolVersion().getCode()); - communityTarget.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); - communityTarget.setTimeout(profileTransportConfig.getTimeoutMs()); - communityTarget.setRetries(profileTransportConfig.getRetries()); - this.target = communityTarget; - log.info("SNMP target initialized: {}", this.target); + + AbstractTarget target; + + SnmpProtocolVersion protocolVersion = deviceTransportConfig.getProtocolVersion(); + switch (protocolVersion) { + case V1: + CommunityTarget communityTargetV1 = new CommunityTarget(); + communityTargetV1.setSecurityModel(SecurityModel.SECURITY_MODEL_SNMPv1); + communityTargetV1.setSecurityLevel(SecurityLevel.NOAUTH_NOPRIV); + communityTargetV1.setCommunity(new OctetString(deviceTransportConfig.getSecurityName())); + target = communityTargetV1; + break; + case V2C: + CommunityTarget communityTargetV2 = new CommunityTarget(); + communityTargetV2.setSecurityModel(SecurityModel.SECURITY_MODEL_SNMPv2c); + communityTargetV2.setSecurityLevel(SecurityLevel.NOAUTH_NOPRIV); + communityTargetV2.setCommunity(new OctetString(deviceTransportConfig.getSecurityName())); + target = communityTargetV2; + break; + case V3: + USM usm = new USM(); + SecurityModels.getInstance().addSecurityModel(usm); + + OctetString securityName = new OctetString(deviceTransportConfig.getSecurityName()); + OctetString authenticationPassphrase = new OctetString(deviceTransportConfig.getAuthenticationPassphrase()); + OctetString privacyPassphrase = new OctetString(deviceTransportConfig.getPrivacyPassphrase()); + + OID authenticationProtocol = AuthSHA.ID; + OID privacyProtocol = PrivDES.ID; // FIXME: to config + + UsmUser user = new UsmUser(securityName, authenticationProtocol, authenticationPassphrase, privacyProtocol, privacyPassphrase); + snmpTransportService.getSnmp().getUSM().addUser(user); + + UserTarget userTarget = new UserTarget(); + userTarget.setSecurityName(securityName); + userTarget.setSecurityLevel(SecurityLevel.AUTH_PRIV); + + target = userTarget; + break; + default: + throw new UnsupportedOperationException("SNMP protocol version " + protocolVersion + " is not supported"); + } + + target.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); + target.setTimeout(profileTransportConfig.getTimeoutMs()); + target.setRetries(profileTransportConfig.getRetries()); + target.setVersion(protocolVersion.getCode()); + + this.target = target; + log.info("SNMP target initialized: {}", target); } public void close() { @@ -138,6 +204,21 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { + profileTransportConfiguration.getCommunicationConfigs().stream() + .filter(config -> config.getSpec() == SnmpCommunicationSpec.SHARED_ATTRIBUTES_SETTING) + .findFirst() + .ifPresent(communicationConfig -> { + Map sharedAttributes = JsonConverter.toJson(attributeUpdateNotification).entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().isJsonPrimitive() ? entry.getValue().getAsString() : entry.getValue().toString() + )); + try { + snmpTransportService.sendRequest(this, communicationConfig, sharedAttributes); + } catch (Exception e) { + log.error("Failed to send request with shared attributes to SNMP device {}: {}", getDeviceId(), e.getMessage()); + } + }); } @Override diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV2.java similarity index 69% rename from common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java rename to common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV2.java index 29fa35dcab..d48eaadce3 100644 --- a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulator.java +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV2.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.transport.snmp; +import org.snmp4j.CommandResponderEvent; import org.snmp4j.CommunityTarget; +import org.snmp4j.PDU; +import org.snmp4j.Snmp; import org.snmp4j.Target; import org.snmp4j.TransportMapping; import org.snmp4j.agent.BaseAgent; @@ -42,45 +45,81 @@ import org.snmp4j.smi.GenericAddress; import org.snmp4j.smi.Integer32; import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.UdpAddress; import org.snmp4j.smi.Variable; +import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.TransportMappings; import java.io.File; import java.io.IOException; import java.util.Map; import java.util.Scanner; +import java.util.function.Consumer; +import java.util.stream.Collectors; -public class SnmpDeviceSimulator extends BaseAgent { +public class SnmpDeviceSimulatorV2 extends BaseAgent { + + public static class RequestProcessor extends CommandProcessor { + private final Consumer processor; + + public RequestProcessor(Consumer processor) { + super(new OctetString(MPv3.createLocalEngineID())); + this.processor = processor; + } + + @Override + public void processPdu(CommandResponderEvent event) { + processor.accept(event); + } + } public static void main(String[] args) throws IOException { - SnmpDeviceSimulator device = new SnmpDeviceSimulator(1610); + SnmpDeviceSimulatorV2 device = new SnmpDeviceSimulatorV2(1610, "public"); device.start(); device.setUpMappings(Map.of( ".1.3.6.1.2.1.1.1.50", "12", ".1.3.6.1.2.1.2.1.52", "56", - ".1.3.6.1.2.1.3.1.54", "yes" + ".1.3.6.1.2.1.3.1.54", "yes", + ".1.3.6.1.2.1.7.1.58", "" )); + +// while (true) { +// new Scanner(System.in).nextLine(); +// device.sendTrap("127.0.0.1", 1062, Map.of(".1.3.6.1.2.87.1.56", "12")); +// System.out.println("sent"); +// } + +// Snmp snmp = new Snmp(device.transportMappings[0]); +// device.snmp.addCommandResponder(event -> { +// System.out.println(event); +// }); + new Scanner(System.in).nextLine(); } private final Target target; private final Address address; + private Snmp snmp; - public SnmpDeviceSimulator(int port) { - super(new File("conf.agent"), new File("bootCounter.agent"), - new CommandProcessor(new OctetString(MPv3.createLocalEngineID()))); + private final String password; + public SnmpDeviceSimulatorV2(int port, String password) throws IOException { + super(new File("conf.agent"), new File("bootCounter.agent"), new RequestProcessor(event -> { + System.out.println("aboba"); + ((Snmp) event.getSource()).cancel(event.getPDU(), event1 -> System.out.println("canceled")); + })); CommunityTarget target = new CommunityTarget(); - target.setCommunity(new OctetString("public")); - address = GenericAddress.parse("udp:0.0.0.0/" + port); + target.setCommunity(new OctetString(password)); + this.address = GenericAddress.parse("udp:0.0.0.0/" + port); target.setAddress(address); target.setRetries(2); target.setTimeout(1500); target.setVersion(SnmpConstants.version2c); this.target = target; + this.password = password; } public void start() throws IOException { @@ -90,17 +129,27 @@ public class SnmpDeviceSimulator extends BaseAgent { finishInit(); run(); sendColdStartNotification(); + snmp = new Snmp(transportMappings[0]); } public void setUpMappings(Map oidToResponseMappings) { unregisterManagedObject(getSnmpv2MIB()); oidToResponseMappings.forEach((oid, response) -> { - registerManagedObject(new MOScalar<>(new OID(oid), MOAccessImpl.ACCESS_READ_ONLY, new OctetString(response))); + registerManagedObject(new MOScalar<>(new OID(oid), MOAccessImpl.ACCESS_READ_WRITE, new OctetString(response))); }); } - public Target getTarget() { - return target; + public void sendTrap(String host, int port, Map values) throws IOException { + PDU pdu = new PDU(); + pdu.addAll(values.entrySet().stream() + .map(entry -> new VariableBinding(new OID(entry.getKey()), new OctetString(entry.getValue()))) + .collect(Collectors.toList())); + pdu.setType(PDU.TRAP); + + CommunityTarget remoteTarget = (CommunityTarget) getTarget().clone(); + remoteTarget.setAddress(new UdpAddress(host + "/" + port)); + + snmp.send(pdu, remoteTarget); } @Override @@ -165,4 +214,9 @@ public class SnmpDeviceSimulator extends BaseAgent { new OctetString("public2public").toSubIndex(true), com2sec); communityMIB.getSnmpCommunityEntry().addRow(row); } + + public Target getTarget() { + return target; + } + } diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java new file mode 100644 index 0000000000..63ccd7021a --- /dev/null +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java @@ -0,0 +1,52 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import org.snmp4j.UserTarget; +import org.snmp4j.security.AuthSHA; +import org.snmp4j.security.PrivDES; +import org.snmp4j.security.SecurityLevel; +import org.snmp4j.security.SecurityModels; +import org.snmp4j.security.USM; +import org.snmp4j.security.UsmUser; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.OctetString; + +import java.io.IOException; + +public class SnmpDeviceSimulatorV3 extends SnmpDeviceSimulatorV2 { + public SnmpDeviceSimulatorV3(int port, String securityName, String authenticationPassphrase, String privacyPassphrase) throws IOException { + super(12, null); +// super(new File("conf.agent"), new File("bootCounter.agent")); + + USM usm = new USM(); + SecurityModels.getInstance().addSecurityModel(usm); + + OID authenticationProtocol = AuthSHA.ID; + OID privacyProtocol = PrivDES.ID; + + UsmUser user = new UsmUser(new OctetString(securityName), authenticationProtocol, new OctetString(authenticationPassphrase), privacyProtocol, new OctetString(privacyPassphrase)); + } + + public void initV3(UsmUser user, String securityName) { +// snmp.getUSM().addUser(user); + + UserTarget userTarget = new UserTarget(); + userTarget.setSecurityName(new OctetString(securityName)); + userTarget.setSecurityLevel(SecurityLevel.AUTH_PRIV); + } + +} diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTest.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTest.java new file mode 100644 index 0000000000..29a8c085c8 --- /dev/null +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTest.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import java.io.IOException; +import java.util.Map; +import java.util.Scanner; + +public class SnmpTest { + public static void main(String[] args) throws IOException { + SnmpDeviceSimulatorV2 device = new SnmpDeviceSimulatorV2(1610, "public"); + + device.start(); + device.setUpMappings(Map.of( + ".1.3.6.1.2.1.1.1.50", "12", + ".1.3.6.1.2.1.2.1.52", "56", + ".1.3.6.1.2.1.3.1.54", "yes", + ".1.3.6.1.2.1.7.1.58", "" + )); + + +// while (true) { +// new Scanner(System.in).nextLine(); +// device.sendTrap("127.0.0.1", 1062, Map.of(".1.3.6.1.2.87.1.56", "12")); +// System.out.println("sent"); +// } + +// Snmp snmp = new Snmp(device.transportMappings[0]); +// device.snmp.addCommandResponder(event -> { +// System.out.println(event); +// }); + + new Scanner(System.in).nextLine(); + } + +} diff --git a/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json b/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json index 78b66c4c0e..f8291a090b 100644 --- a/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json +++ b/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json @@ -1,27 +1,53 @@ { - "pollPeriodMs": 3000, "timeoutMs": 500, "retries": 0, - "telemetryMappings": [ + "communicationConfigs": [ { - "oid": ".1.3.6.1.2.1.1.1.50", - "method": "GET", - "key": "temperature", - "dataType": "LONG" + "spec": "TELEMETRY_QUERYING", + "queryingFrequencyMs": 3000, + "mappings": [ + { + "oid": ".1.3.6.1.2.1.1.1.50", + "key": "temperature", + "dataType": "LONG" + }, + { + "oid": ".1.3.6.1.2.1.2.1.52", + "key": "humidity", + "dataType": "DOUBLE" + } + ] }, { - "oid": ".1.3.6.1.2.1.2.1.52", - "method": "GET", - "key": "humidity", - "dataType": "LONG" - } - ], - "attributesMappings": [ + "spec": "CLIENT_ATTRIBUTES_QUERYING", + "queryingFrequencyMs": 5000, + "mappings": [ + { + "oid": ".1.3.6.1.2.1.3.1.54", + "key": "isCool", + "dataType": "STRING" + } + ] + }, + { + "spec": "SHARED_ATTRIBUTES_SETTING", + "mappings": [ + { + "oid": ".1.3.6.1.2.1.7.1.58", + "key": "shared", + "dataType": "STRING" + } + ] + }, { - "oid": ".1.3.6.1.2.1.3.1.54", - "method": "GET", - "key": "isCool", - "dataType": "STRING" + "spec": "TELEMETRY_TRAPS_RECEIVING", + "mappings": [ + { + "oid": ".1.3.6.1.2.8.7.1.56", + "key": "temperature_trap", + "dataType": "LONG" + } + ] } ] -} \ No newline at end of file +} diff --git a/common/transport/snmp/src/test/resources/snmp-device-transport-config.json b/common/transport/snmp/src/test/resources/snmp-device-transport-config.json index c73d817bfb..9a97b75e40 100644 --- a/common/transport/snmp/src/test/resources/snmp-device-transport-config.json +++ b/common/transport/snmp/src/test/resources/snmp-device-transport-config.json @@ -1,6 +1,6 @@ { "address": "127.0.0.1", "port": 1610, - "community": "public", + "password": "public", "protocolVersion": "V2C" } \ No newline at end of file diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 2c5dfafe4e..699e43e5ee 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -285,7 +285,7 @@ public class JsonConverter { return result; } - public static JsonElement toJson(AttributeUpdateNotificationMsg payload) { + public static JsonObject toJson(AttributeUpdateNotificationMsg payload) { JsonObject result = new JsonObject(); if (payload.getSharedUpdatedCount() > 0) { payload.getSharedUpdatedList().forEach(addToObjectFromProto(result)); From e52ac96c624fa2f2790d5a288683e675311e8a8a Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 6 Apr 2021 17:05:34 +0300 Subject: [PATCH 10/32] Implement SNMP v3 security support, remove traps support --- .../SnmpDeviceTransportConfiguration.java | 23 +- .../snmp/AuthenticationProtocol.java | 45 ++ .../data/transport/snmp/PrivacyProtocol.java | 43 ++ .../transport/snmp/SnmpCommunicationSpec.java | 5 +- ...TrapsReceivingSnmpCommunicationConfig.java | 25 - .../snmp/configs/SnmpCommunicationConfig.java | 4 +- ...TrapsReceivingSnmpCommunicationConfig.java | 30 - .../transport/snmp/SnmpAuthService.java | 118 +++ .../transport/snmp/SnmpTransportContext.java | 12 +- .../snmp/service/SnmpTransportService.java | 142 ++-- .../snmp/session/DeviceSessionContext.java | 74 +- .../transport/snmp/SnmpDeviceSimulatorV2.java | 26 - .../transport/snmp/SnmpDeviceSimulatorV3.java | 723 +++++++++++++++++- .../snmp/{SnmpTest.java => SnmpTestV2.java} | 2 +- .../server/transport/snmp/SnmpTestV3.java | 46 ++ .../snmp-device-profile-transport-config.json | 10 - .../snmp-device-transport-config-v3.json | 13 + .../snmp-device-transport-config.json | 2 +- 18 files changed, 1057 insertions(+), 286 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/AuthenticationProtocol.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/PrivacyProtocol.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java rename common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/{SnmpTest.java => SnmpTestV2.java} (98%) create mode 100644 common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTestV3.java create mode 100644 common/transport/snmp/src/test/resources/snmp-device-transport-config-v3.json diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index a6056b4188..24ab463129 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -19,6 +19,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.transport.snmp.AuthenticationProtocol; +import org.thingsboard.server.common.data.transport.snmp.PrivacyProtocol; import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; @Data @@ -26,9 +28,23 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur private String address; private int port; private SnmpProtocolVersion protocolVersion; + + /* + * For SNMP v1 and v2c + * */ + private String community; + + /* + * For SNMP v3 with User Based Security Model + * */ + private String username; private String securityName; - private String authenticationPassphrase; // for SNMP v3 - private String privacyPassphrase; // for SNMP v3 + private String contextName; + private AuthenticationProtocol authenticationProtocol; + private String authenticationPassphrase; + private PrivacyProtocol privacyProtocol; + private String privacyPassphrase; + private String engineId; @Override public DeviceTransportType getType() { @@ -44,7 +60,6 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur @JsonIgnore private boolean isValid() { - return StringUtils.isNotBlank(address) && port > 0 && - StringUtils.isNotBlank(securityName) && protocolVersion != null; + return true; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/AuthenticationProtocol.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/AuthenticationProtocol.java new file mode 100644 index 0000000000..42d2e98798 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/AuthenticationProtocol.java @@ -0,0 +1,45 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp; + +import java.util.Arrays; +import java.util.Optional; + +public enum AuthenticationProtocol { + SHA_1("1.3.6.1.6.3.10.1.1.3"), + SHA_224("1.3.6.1.6.3.10.1.1.4"), + SHA_256("1.3.6.1.6.3.10.1.1.5"), + SHA_384("1.3.6.1.6.3.10.1.1.6"), + SHA_512("1.3.6.1.6.3.10.1.1.7"), + MD5("1.3.6.1.6.3.10.1.1.2"); + + // oids taken from org.snmp4j.security.SecurityProtocol implementations + private final String oid; + + AuthenticationProtocol(String oid) { + this.oid = oid; + } + + public String getOid() { + return oid; + } + + public static Optional forName(String name) { + return Arrays.stream(values()) + .filter(protocol -> protocol.name().equalsIgnoreCase(name)) + .findFirst(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/PrivacyProtocol.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/PrivacyProtocol.java new file mode 100644 index 0000000000..1c040eb83e --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/PrivacyProtocol.java @@ -0,0 +1,43 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp; + +import java.util.Arrays; +import java.util.Optional; + +public enum PrivacyProtocol { + DES("1.3.6.1.6.3.10.1.2.2"), + AES_128("1.3.6.1.6.3.10.1.2.4"), + AES_192("1.3.6.1.4.1.4976.2.2.1.1.1"), + AES_256("1.3.6.1.4.1.4976.2.2.1.1.2"); + + // oids taken from org.snmp4j.security.SecurityProtocol implementations + private final String oid; + + PrivacyProtocol(String oid) { + this.oid = oid; + } + + public String getOid() { + return oid; + } + + public static Optional forName(String name) { + return Arrays.stream(values()) + .filter(protocol -> protocol.name().equalsIgnoreCase(name)) + .findFirst(); + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java index f6ff7d3109..a2edf75968 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java @@ -19,10 +19,7 @@ public enum SnmpCommunicationSpec { TELEMETRY_QUERYING(true), CLIENT_ATTRIBUTES_QUERYING(true), - SHARED_ATTRIBUTES_SETTING, - - TELEMETRY_TRAPS_RECEIVING, - CLIENT_ATTRIBUTES_TRAPS_RECEIVING; + SHARED_ATTRIBUTES_SETTING; private final boolean isRepeatingQuerying; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java deleted file mode 100644 index 9a11b01f1b..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesTrapsReceivingSnmpCommunicationConfig.java +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.transport.snmp.configs; - -import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; - -public class ClientAttributesTrapsReceivingSnmpCommunicationConfig extends SnmpCommunicationConfig { - @Override - public SnmpCommunicationSpec getSpec() { - return SnmpCommunicationSpec.CLIENT_ATTRIBUTES_TRAPS_RECEIVING; - } -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java index bfc194669b..7e1ee27237 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java @@ -31,9 +31,7 @@ import java.util.List; @JsonSubTypes({ @Type(value = TelemetryQueryingSnmpCommunicationConfig.class, name = "TELEMETRY_QUERYING"), @Type(value = ClientAttributesQueryingSnmpCommunicationConfig.class, name = "CLIENT_ATTRIBUTES_QUERYING"), - @Type(value = SharedAttributesSettingSnmpCommunicationConfig.class, name = "SHARED_ATTRIBUTES_SETTING"), - @Type(value = TelemetryTrapsReceivingSnmpCommunicationConfig.class, name = "TELEMETRY_TRAPS_RECEIVING"), - @Type(value = ClientAttributesTrapsReceivingSnmpCommunicationConfig.class, name = "CLIENT_ATTRIBUTES_TRAPS_RECEIVING") + @Type(value = SharedAttributesSettingSnmpCommunicationConfig.class, name = "SHARED_ATTRIBUTES_SETTING") }) public abstract class SnmpCommunicationConfig { protected List mappings; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java deleted file mode 100644 index 66be4d0cf8..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryTrapsReceivingSnmpCommunicationConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.transport.snmp.configs; - -import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; - -public class TelemetryTrapsReceivingSnmpCommunicationConfig extends SnmpCommunicationConfig { - @Override - public SnmpCommunicationSpec getSpec() { - return SnmpCommunicationSpec.TELEMETRY_TRAPS_RECEIVING; - } - - @Override - public boolean isValid() { - return false; - } -} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java new file mode 100644 index 0000000000..59b31d593c --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java @@ -0,0 +1,118 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import lombok.RequiredArgsConstructor; +import org.snmp4j.AbstractTarget; +import org.snmp4j.CommunityTarget; +import org.snmp4j.Target; +import org.snmp4j.UserTarget; +import org.snmp4j.security.SecurityLevel; +import org.snmp4j.security.SecurityModel; +import org.snmp4j.security.SecurityProtocols; +import org.snmp4j.security.USM; +import org.snmp4j.security.UsmUser; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.OctetString; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.service.SnmpTransportService; +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; + +@Service +@TbSnmpTransportComponent +@RequiredArgsConstructor +public class SnmpAuthService { + private final SnmpTransportService snmpTransportService; + + @Value("${transport.snmp.underlying_protocol}") + private String snmpUnderlyingProtocol; + + public Target setUpSnmpTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + AbstractTarget target; + + SnmpProtocolVersion protocolVersion = deviceTransportConfig.getProtocolVersion(); + switch (protocolVersion) { + case V1: + CommunityTarget communityTargetV1 = new CommunityTarget(); + communityTargetV1.setSecurityModel(SecurityModel.SECURITY_MODEL_SNMPv1); + communityTargetV1.setSecurityLevel(SecurityLevel.NOAUTH_NOPRIV); + communityTargetV1.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); + target = communityTargetV1; + break; + case V2C: + CommunityTarget communityTargetV2 = new CommunityTarget(); + communityTargetV2.setSecurityModel(SecurityModel.SECURITY_MODEL_SNMPv2c); + communityTargetV2.setSecurityLevel(SecurityLevel.NOAUTH_NOPRIV); + communityTargetV2.setCommunity(new OctetString(deviceTransportConfig.getCommunity())); + target = communityTargetV2; + break; + case V3: + OctetString username = new OctetString(deviceTransportConfig.getUsername()); + OctetString securityName = new OctetString(deviceTransportConfig.getSecurityName()); + OctetString engineId = new OctetString(deviceTransportConfig.getEngineId()); + + OID authenticationProtocol = new OID(deviceTransportConfig.getAuthenticationProtocol().getOid()); + OID privacyProtocol = new OID(deviceTransportConfig.getPrivacyProtocol().getOid()); + OctetString authenticationPassphrase = new OctetString(deviceTransportConfig.getAuthenticationPassphrase()); + authenticationPassphrase = new OctetString(SecurityProtocols.getInstance().passwordToKey(authenticationProtocol, authenticationPassphrase, engineId.getValue())); + OctetString privacyPassphrase = new OctetString(deviceTransportConfig.getPrivacyPassphrase()); + privacyPassphrase = new OctetString(SecurityProtocols.getInstance().passwordToKey(privacyProtocol, authenticationProtocol, privacyPassphrase, engineId.getValue())); + + USM usm = snmpTransportService.getSnmp().getUSM(); + if (usm.hasUser(engineId, securityName)) { + usm.removeAllUsers(username, engineId); + } + usm.addLocalizedUser( + engineId.getValue(), username, + authenticationProtocol, authenticationPassphrase.getValue(), + privacyProtocol, privacyPassphrase.getValue() + ); + + UserTarget userTarget = new UserTarget(); + userTarget.setSecurityName(securityName); + userTarget.setAuthoritativeEngineID(engineId.getValue()); + userTarget.setSecurityModel(SecurityModel.SECURITY_MODEL_USM); + userTarget.setSecurityLevel(SecurityLevel.AUTH_PRIV); + target = userTarget; + break; + default: + throw new UnsupportedOperationException("SNMP protocol version " + protocolVersion + " is not supported"); + } + + target.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); + target.setTimeout(profileTransportConfig.getTimeoutMs()); + target.setRetries(profileTransportConfig.getRetries()); + target.setVersion(protocolVersion.getCode()); + + return target; + } + + public void cleanUpSnmpAuthInfo(DeviceSessionContext sessionContext) { + SnmpDeviceTransportConfiguration deviceTransportConfiguration = sessionContext.getDeviceTransportConfiguration(); + if (deviceTransportConfiguration.getProtocolVersion() == SnmpProtocolVersion.V3) { + OctetString username = new OctetString(deviceTransportConfiguration.getUsername()); + OctetString engineId = new OctetString(deviceTransportConfiguration.getEngineId()); + snmpTransportService.getSnmp().getUSM().removeAllUsers(username, engineId); + } + } + +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 877d4f1652..2b0ce984aa 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -15,9 +15,9 @@ */ package org.thingsboard.server.transport.snmp; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Value; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Device; @@ -61,18 +61,18 @@ import java.util.stream.Collectors; @Slf4j @RequiredArgsConstructor public class SnmpTransportContext extends TransportContext { + @Getter private final SnmpTransportService snmpTransportService; private final TransportDeviceProfileCache deviceProfileCache; private final TransportService transportService; private final ProtoTransportEntityService protoEntityService; private final SnmpTransportBalancingService balancingService; + @Getter + private final SnmpAuthService snmpAuthService; private final Map sessions = new ConcurrentHashMap<>(); private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); - @Value("${transport.snmp.underlying_protocol}") - private String snmpUnderlyingProtocol; - @AfterStartUp(order = 2) public void initDevicesSessions() { log.info("Initializing SNMP devices sessions"); @@ -116,8 +116,7 @@ public class SnmpTransportContext extends TransportContext { DeviceSessionContext deviceSessionContext = new DeviceSessionContext( device, deviceProfile, credentials.getCredentialsId(), - profileTransportConfiguration, deviceTransportConfiguration, - this, snmpTransportService, snmpUnderlyingProtocol + profileTransportConfiguration, deviceTransportConfiguration, this ); registerSessionMsgListener(deviceSessionContext); sessions.put(device.getId(), deviceSessionContext); @@ -155,6 +154,7 @@ public class SnmpTransportContext extends TransportContext { if (sessionContext == null) return; log.info("Destroying SNMP device session for device {}", sessionContext.getDevice().getId()); sessionContext.close(); + snmpAuthService.cleanUpSnmpAuthInfo(sessionContext); transportService.deregisterSession(sessionContext.getSessionInfo()); sessions.remove(sessionContext.getDeviceId()); snmpTransportService.cancelQueryingTasks(sessionContext); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 963b5db382..2f26bfa98c 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -19,29 +19,31 @@ import com.google.gson.JsonObject; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.snmp4j.CommandResponder; -import org.snmp4j.CommandResponderEvent; import org.snmp4j.PDU; +import org.snmp4j.ScopedPDU; import org.snmp4j.Snmp; import org.snmp4j.TransportMapping; import org.snmp4j.event.ResponseEvent; +import org.snmp4j.mp.MPv3; +import org.snmp4j.security.SecurityModels; +import org.snmp4j.security.SecurityProtocols; +import org.snmp4j.security.USM; import org.snmp4j.smi.Null; import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; -import org.snmp4j.smi.TcpAddress; -import org.snmp4j.smi.UdpAddress; import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultTcpTransportMapping; import org.snmp4j.transport.DefaultUdpTransportMapping; import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.TbTransportService; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; import org.thingsboard.server.common.data.transport.snmp.configs.RepeatingQueryingSnmpCommunicationConfig; import org.thingsboard.server.common.data.transport.snmp.configs.SnmpCommunicationConfig; import org.thingsboard.server.common.transport.TransportService; @@ -49,12 +51,12 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; -import org.thingsboard.server.transport.snmp.SnmpTransportContext; import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; +import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.List; @@ -67,13 +69,11 @@ import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; import java.util.function.BiConsumer; import java.util.stream.Collectors; -import java.util.stream.Stream; @TbSnmpTransportComponent @Service @Slf4j -public class SnmpTransportService implements TbTransportService, CommandResponder { - private final SnmpTransportContext snmpTransportContext; +public class SnmpTransportService implements TbTransportService { private final TransportService transportService; @Getter @@ -88,9 +88,7 @@ public class SnmpTransportService implements TbTransportService, CommandResponde @Value("${transport.snmp.underlying_protocol}") private String snmpUnderlyingProtocol; - public SnmpTransportService(@Lazy SnmpTransportContext snmpTransportContext, - TransportService transportService) { - this.snmpTransportContext = snmpTransportContext; + public SnmpTransportService(TransportService transportService) { this.transportService = transportService; } @@ -102,7 +100,6 @@ public class SnmpTransportService implements TbTransportService, CommandResponde responseProcessingExecutor = Executors.newWorkStealingPool(responseProcessingParallelismLevel); initializeSnmp(); - initializeTrapsListener(); configureResponseProcessors(); log.info("SNMP transport service initialized"); @@ -122,29 +119,9 @@ public class SnmpTransportService implements TbTransportService, CommandResponde } snmp = new Snmp(transportMapping); snmp.listen(); - } - - private void initializeTrapsListener() throws IOException { - int trapsListeningPort = 1062; - String bindingAddress = "0.0.0.0/" + trapsListeningPort; - TransportMapping transportMapping; - switch (snmpUnderlyingProtocol) { - case "udp": - transportMapping = new DefaultUdpTransportMapping(new UdpAddress(bindingAddress)); - break; - case "tcp": - transportMapping = new DefaultTcpTransportMapping(new TcpAddress(bindingAddress)); - break; - default: - throw new IllegalArgumentException("Underlying protocol " + snmpUnderlyingProtocol + " for SNMP is not supported"); - } - - - Snmp trapsSnmp = new Snmp(transportMapping); - trapsSnmp.addCommandResponder(this); - - transportMapping.listen(); + USM usm = new USM(SecurityProtocols.getInstance(), new OctetString(MPv3.createLocalEngineID()), 0); + SecurityModels.getInstance().addSecurityModel(usm); } public void createQueryingTasks(DeviceSessionContext sessionContext) { @@ -177,12 +154,11 @@ public class SnmpTransportService implements TbTransportService, CommandResponde } public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) throws IOException { - PDU request = createPdu(communicationConfig); - executeRequest(sessionContext, request); + sendRequest(sessionContext, communicationConfig, Collections.emptyMap()); } public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) throws IOException { - PDU request = createPduWithValues(communicationConfig, values); + PDU request = createPdu(sessionContext, communicationConfig, values); executeRequest(sessionContext, request); } @@ -193,46 +169,37 @@ public class SnmpTransportService implements TbTransportService, CommandResponde } } - private PDU createPdu(SnmpCommunicationConfig communicationConfig) { - PDU pdu = new PDU(); - pdu.setType(communicationConfig.getMethod().getCode()); - pdu.addAll(communicationConfig.getMappings().stream() - .map(mapping -> new VariableBinding(new OID(mapping.getOid()))) - .collect(Collectors.toList())); - return pdu; - } + private PDU createPdu(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) { + PDU pdu; + SnmpDeviceTransportConfiguration deviceTransportConfiguration = sessionContext.getDeviceTransportConfiguration(); + SnmpProtocolVersion snmpVersion = deviceTransportConfiguration.getProtocolVersion(); + switch (snmpVersion) { + case V1: + case V2C: + pdu = new PDU(); + break; + case V3: + ScopedPDU scopedPdu = new ScopedPDU(); + scopedPdu.setContextName(new OctetString(deviceTransportConfiguration.getContextName())); + scopedPdu.setContextEngineID(new OctetString(deviceTransportConfiguration.getEngineId())); + pdu = scopedPdu; + break; + default: + throw new UnsupportedOperationException("SNMP version " + snmpVersion + " is not supported"); + } - private PDU createPduWithValues(SnmpCommunicationConfig communicationConfig, Map values) { - PDU pdu = new PDU(); pdu.setType(communicationConfig.getMethod().getCode()); pdu.addAll(communicationConfig.getMappings().stream() - .filter(mapping -> values.containsKey(mapping.getKey())) - .map(mapping -> { - String value = values.get(mapping.getKey()); - return new VariableBinding(new OID(mapping.getOid()), new OctetString(value)); - }) + .filter(mapping -> values.isEmpty() || values.containsKey(mapping.getKey())) + .map(mapping -> Optional.ofNullable(values.get(mapping.getKey())) + .map(value -> new VariableBinding(new OID(mapping.getOid()), new OctetString(values.get(mapping.getKey())))) + .orElseGet(() -> new VariableBinding(new OID(mapping.getOid())))) .collect(Collectors.toList())); + return pdu; } - private void processTrap(CommandResponderEvent event) { - if (event.getPDU().getType() != PDU.TRAP) return; - - snmpTransportContext.getSessions().stream() - .filter(sessionContext -> { - // TODO: SNMP v3 support - return sessionContext.getTarget().getSecurityName().equals(OctetString.fromByteArray(event.getSecurityName())) && - sessionContext.getTarget().getAddress().equals(event.getPeerAddress()); - }) - .findFirst() - .ifPresentOrElse(sessionContext -> { - responseProcessingExecutor.execute(() -> processResponse(sessionContext, event.getPDU())); - }, () -> { - log.debug("SNMP event is from unknown source: {}", event); - }); - } - public void processResponseEvent(DeviceSessionContext sessionContext, ResponseEvent event) { ((Snmp) event.getSource()).cancel(event.getRequest(), sessionContext); @@ -243,7 +210,7 @@ public class SnmpTransportService implements TbTransportService, CommandResponde PDU response = event.getResponse(); if (response == null) { - log.warn("No response from SNMP device {}, requestId: {}", sessionContext.getDeviceId(), event.getRequest().getRequestID()); + log.debug("No response from SNMP device {}, requestId: {}", sessionContext.getDeviceId(), event.getRequest().getRequestID()); return; } DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); @@ -308,23 +275,17 @@ public class SnmpTransportService implements TbTransportService, CommandResponde } private void configureResponseProcessors() { - Stream.of(SnmpCommunicationSpec.TELEMETRY_QUERYING, SnmpCommunicationSpec.TELEMETRY_TRAPS_RECEIVING) - .forEach(telemetrySpec -> { - responseProcessors.put(telemetrySpec, (response, sessionContext) -> { - TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(response); - transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, TransportServiceCallback.EMPTY); - log.debug("Posted telemetry for device {}: {}", sessionContext.getDeviceId(), response); - }); - }); - - Stream.of(SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING, SnmpCommunicationSpec.CLIENT_ATTRIBUTES_TRAPS_RECEIVING) - .forEach(clientAttributesSpec -> { - responseProcessors.put(clientAttributesSpec, (response, sessionContext) -> { - TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(response); - transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, TransportServiceCallback.EMPTY); - log.debug("Posted attributes for device {}: {}", sessionContext.getDeviceId(), response); - }); - }); + responseProcessors.put(SnmpCommunicationSpec.TELEMETRY_QUERYING, (response, sessionContext) -> { + TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(response); + transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, TransportServiceCallback.EMPTY); + log.debug("Posted telemetry for device {}: {}", sessionContext.getDeviceId(), response); + }); + + responseProcessors.put(SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING, (response, sessionContext) -> { + TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(response); + transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, TransportServiceCallback.EMPTY); + log.debug("Posted attributes for device {}: {}", sessionContext.getDeviceId(), response); + }); } private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { @@ -353,11 +314,6 @@ public class SnmpTransportService implements TbTransportService, CommandResponde } } - @Override - public void processPdu(CommandResponderEvent event) { - processTrap(event); - } - @Override public String getName() { return "SNMP"; diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index c0bbc843bc..c5a329547f 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -18,29 +18,15 @@ package org.thingsboard.server.transport.snmp.session; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.snmp4j.AbstractTarget; -import org.snmp4j.CommunityTarget; import org.snmp4j.Target; -import org.snmp4j.UserTarget; import org.snmp4j.event.ResponseEvent; import org.snmp4j.event.ResponseListener; -import org.snmp4j.security.AuthSHA; -import org.snmp4j.security.PrivDES; -import org.snmp4j.security.SecurityLevel; -import org.snmp4j.security.SecurityModel; -import org.snmp4j.security.SecurityModels; -import org.snmp4j.security.USM; -import org.snmp4j.security.UsmUser; -import org.snmp4j.smi.GenericAddress; -import org.snmp4j.smi.OID; -import org.snmp4j.smi.OctetString; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; -import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; @@ -50,6 +36,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponse import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; +import org.thingsboard.server.transport.snmp.SnmpAuthService; import org.thingsboard.server.transport.snmp.SnmpTransportContext; import org.thingsboard.server.transport.snmp.service.SnmpTransportService; @@ -77,6 +64,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S private final SnmpTransportContext snmpTransportContext; private final SnmpTransportService snmpTransportService; + private final SnmpAuthService snmpAuthService; @Getter @Setter @@ -84,7 +72,6 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S private final AtomicInteger msgIdSeq = new AtomicInteger(0); @Getter private boolean isActive = true; - private final String snmpUnderlyingProtocol; @Getter @Setter @@ -93,8 +80,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S public DeviceSessionContext(Device device, DeviceProfile deviceProfile, String token, SnmpDeviceProfileTransportConfiguration profileTransportConfiguration, SnmpDeviceTransportConfiguration deviceTransportConfiguration, - SnmpTransportContext snmpTransportContext, SnmpTransportService snmpTransportService, - String snmpUnderlyingProtocol) { + SnmpTransportContext snmpTransportContext) { super(UUID.randomUUID()); super.setDeviceId(device.getId()); super.setDeviceProfile(deviceProfile); @@ -102,12 +88,12 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S this.token = token; this.snmpTransportContext = snmpTransportContext; - this.snmpTransportService = snmpTransportService; + this.snmpTransportService = snmpTransportContext.getSnmpTransportService(); + this.snmpAuthService = snmpTransportContext.getSnmpAuthService(); this.profileTransportConfiguration = profileTransportConfiguration; this.deviceTransportConfiguration = deviceTransportConfiguration; - this.snmpUnderlyingProtocol = snmpUnderlyingProtocol; initializeTarget(profileTransportConfiguration, deviceTransportConfiguration); } @@ -133,55 +119,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S public void initializeTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { log.trace("Initializing target for SNMP session of device {}", device); - - AbstractTarget target; - - SnmpProtocolVersion protocolVersion = deviceTransportConfig.getProtocolVersion(); - switch (protocolVersion) { - case V1: - CommunityTarget communityTargetV1 = new CommunityTarget(); - communityTargetV1.setSecurityModel(SecurityModel.SECURITY_MODEL_SNMPv1); - communityTargetV1.setSecurityLevel(SecurityLevel.NOAUTH_NOPRIV); - communityTargetV1.setCommunity(new OctetString(deviceTransportConfig.getSecurityName())); - target = communityTargetV1; - break; - case V2C: - CommunityTarget communityTargetV2 = new CommunityTarget(); - communityTargetV2.setSecurityModel(SecurityModel.SECURITY_MODEL_SNMPv2c); - communityTargetV2.setSecurityLevel(SecurityLevel.NOAUTH_NOPRIV); - communityTargetV2.setCommunity(new OctetString(deviceTransportConfig.getSecurityName())); - target = communityTargetV2; - break; - case V3: - USM usm = new USM(); - SecurityModels.getInstance().addSecurityModel(usm); - - OctetString securityName = new OctetString(deviceTransportConfig.getSecurityName()); - OctetString authenticationPassphrase = new OctetString(deviceTransportConfig.getAuthenticationPassphrase()); - OctetString privacyPassphrase = new OctetString(deviceTransportConfig.getPrivacyPassphrase()); - - OID authenticationProtocol = AuthSHA.ID; - OID privacyProtocol = PrivDES.ID; // FIXME: to config - - UsmUser user = new UsmUser(securityName, authenticationProtocol, authenticationPassphrase, privacyProtocol, privacyPassphrase); - snmpTransportService.getSnmp().getUSM().addUser(user); - - UserTarget userTarget = new UserTarget(); - userTarget.setSecurityName(securityName); - userTarget.setSecurityLevel(SecurityLevel.AUTH_PRIV); - - target = userTarget; - break; - default: - throw new UnsupportedOperationException("SNMP protocol version " + protocolVersion + " is not supported"); - } - - target.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); - target.setTimeout(profileTransportConfig.getTimeoutMs()); - target.setRetries(profileTransportConfig.getRetries()); - target.setVersion(protocolVersion.getCode()); - - this.target = target; + this.target = snmpAuthService.setUpSnmpTarget(profileTransportConfig, deviceTransportConfig); log.info("SNMP target initialized: {}", target); } diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV2.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV2.java index d48eaadce3..c97f47d573 100644 --- a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV2.java +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV2.java @@ -73,32 +73,6 @@ public class SnmpDeviceSimulatorV2 extends BaseAgent { } } - public static void main(String[] args) throws IOException { - SnmpDeviceSimulatorV2 device = new SnmpDeviceSimulatorV2(1610, "public"); - - device.start(); - device.setUpMappings(Map.of( - ".1.3.6.1.2.1.1.1.50", "12", - ".1.3.6.1.2.1.2.1.52", "56", - ".1.3.6.1.2.1.3.1.54", "yes", - ".1.3.6.1.2.1.7.1.58", "" - )); - - -// while (true) { -// new Scanner(System.in).nextLine(); -// device.sendTrap("127.0.0.1", 1062, Map.of(".1.3.6.1.2.87.1.56", "12")); -// System.out.println("sent"); -// } - -// Snmp snmp = new Snmp(device.transportMappings[0]); -// device.snmp.addCommandResponder(event -> { -// System.out.println(event); -// }); - - new Scanner(System.in).nextLine(); - } - private final Target target; private final Address address; diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java index 63ccd7021a..da7f984267 100644 --- a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpDeviceSimulatorV3.java @@ -15,38 +15,731 @@ */ package org.thingsboard.server.transport.snmp; -import org.snmp4j.UserTarget; +import org.snmp4j.MessageDispatcherImpl; +import org.snmp4j.TransportMapping; +import org.snmp4j.agent.BaseAgent; +import org.snmp4j.agent.CommandProcessor; +import org.snmp4j.agent.DuplicateRegistrationException; +import org.snmp4j.agent.MOGroup; +import org.snmp4j.agent.ManagedObject; +import org.snmp4j.agent.mo.DefaultMOMutableRow2PC; +import org.snmp4j.agent.mo.DefaultMOTable; +import org.snmp4j.agent.mo.MOAccessImpl; +import org.snmp4j.agent.mo.MOColumn; +import org.snmp4j.agent.mo.MOMutableColumn; +import org.snmp4j.agent.mo.MOMutableTableModel; +import org.snmp4j.agent.mo.MOScalar; +import org.snmp4j.agent.mo.MOTableIndex; +import org.snmp4j.agent.mo.MOTableRow; +import org.snmp4j.agent.mo.MOTableSubIndex; +import org.snmp4j.agent.mo.ext.AgentppSimulationMib; +import org.snmp4j.agent.mo.snmp.RowStatus; +import org.snmp4j.agent.mo.snmp.SnmpCommunityMIB; +import org.snmp4j.agent.mo.snmp.SnmpNotificationMIB; +import org.snmp4j.agent.mo.snmp.SnmpTargetMIB; +import org.snmp4j.agent.mo.snmp.StorageType; +import org.snmp4j.agent.mo.snmp.TransportDomains; +import org.snmp4j.agent.mo.snmp.VacmMIB; +import org.snmp4j.agent.mo.snmp4j.example.Snmp4jHeartbeatMib; +import org.snmp4j.agent.security.MutableVACM; +import org.snmp4j.mp.MPv1; +import org.snmp4j.mp.MPv2c; +import org.snmp4j.mp.MPv3; +import org.snmp4j.mp.MessageProcessingModel; +import org.snmp4j.security.AuthHMAC192SHA256; +import org.snmp4j.security.AuthMD5; import org.snmp4j.security.AuthSHA; +import org.snmp4j.security.PrivAES128; +import org.snmp4j.security.PrivAES192; +import org.snmp4j.security.PrivAES256; import org.snmp4j.security.PrivDES; import org.snmp4j.security.SecurityLevel; +import org.snmp4j.security.SecurityModel; import org.snmp4j.security.SecurityModels; +import org.snmp4j.security.SecurityProtocols; import org.snmp4j.security.USM; import org.snmp4j.security.UsmUser; +import org.snmp4j.smi.Address; +import org.snmp4j.smi.Gauge32; +import org.snmp4j.smi.GenericAddress; +import org.snmp4j.smi.Integer32; import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.SMIConstants; +import org.snmp4j.smi.TcpAddress; +import org.snmp4j.smi.TimeTicks; +import org.snmp4j.smi.UdpAddress; +import org.snmp4j.smi.Variable; +import org.snmp4j.transport.DefaultTcpTransportMapping; +import org.snmp4j.transport.TransportMappings; +import org.snmp4j.util.ThreadPool; +import java.io.File; import java.io.IOException; +import java.util.Map; -public class SnmpDeviceSimulatorV3 extends SnmpDeviceSimulatorV2 { - public SnmpDeviceSimulatorV3(int port, String securityName, String authenticationPassphrase, String privacyPassphrase) throws IOException { - super(12, null); -// super(new File("conf.agent"), new File("bootCounter.agent")); +/** + * The TestAgent is a sample SNMP agent implementation of all + * features (MIB implementations) provided by the SNMP4J-Agent framework. + * + * Note, for snmp4s, this code is mostly a copy from snmp4j. + * And don't remove snmp users + * + */ +public class SnmpDeviceSimulatorV3 extends BaseAgent { + protected String address; + private Snmp4jHeartbeatMib heartbeatMIB; + private AgentppSimulationMib agentppSimulationMIB; + + public SnmpDeviceSimulatorV3(CommandProcessor processor) throws IOException { + super(new File("SNMP4JTestAgentBC.cfg"), new File("SNMP4JTestAgentConfig.cfg"), + processor); + agent.setWorkerPool(ThreadPool.create("RequestPool", 4)); + } + + public void setUpMappings(Map oidToResponseMappings) { + unregisterManagedObject(getSnmpv2MIB()); + oidToResponseMappings.forEach((oid, response) -> { + registerManagedObject(new MOScalar<>(new OID(oid), MOAccessImpl.ACCESS_READ_WRITE, new OctetString(response))); + }); + } + protected void registerManagedObject(ManagedObject mo) { + try { + server.register(mo, null); + } catch (DuplicateRegistrationException ex) { + throw new RuntimeException(ex); + } + } + + protected void unregisterManagedObject(MOGroup moGroup) { + moGroup.unregisterMOs(server, getContext(moGroup)); + } + + protected void registerManagedObjects() { + try { + server.register(createStaticIfTable(), null); + server.register(createStaticIfXTable(), null); + agentppSimulationMIB.registerMOs(server, null); + heartbeatMIB.registerMOs(server, null); + } catch (DuplicateRegistrationException ex) { + ex.printStackTrace(); + } + } + + protected void addNotificationTargets(SnmpTargetMIB targetMIB, + SnmpNotificationMIB notificationMIB) { + targetMIB.addDefaultTDomains(); + + targetMIB.addTargetAddress(new OctetString("notificationV2c"), + TransportDomains.transportDomainUdpIpv4, + new OctetString(new UdpAddress("127.0.0.1/162").getValue()), + 200, 1, + new OctetString("notify"), + new OctetString("v2c"), + StorageType.permanent); + targetMIB.addTargetAddress(new OctetString("notificationV3"), + TransportDomains.transportDomainUdpIpv4, + new OctetString(new UdpAddress("127.0.0.1/1162").getValue()), + 200, 1, + new OctetString("notify"), + new OctetString("v3notify"), + StorageType.permanent); + targetMIB.addTargetParams(new OctetString("v2c"), + MessageProcessingModel.MPv2c, + SecurityModel.SECURITY_MODEL_SNMPv2c, + new OctetString("cpublic"), + SecurityLevel.AUTH_PRIV, + StorageType.permanent); + targetMIB.addTargetParams(new OctetString("v3notify"), + MessageProcessingModel.MPv3, + SecurityModel.SECURITY_MODEL_USM, + new OctetString("v3notify"), + SecurityLevel.NOAUTH_NOPRIV, + StorageType.permanent); + notificationMIB.addNotifyEntry(new OctetString("default"), + new OctetString("notify"), + SnmpNotificationMIB.SnmpNotifyTypeEnum.inform, + StorageType.permanent); + } + protected void addViews(VacmMIB vacm) { + vacm.addGroup(SecurityModel.SECURITY_MODEL_SNMPv1, + new OctetString("cpublic"), + new OctetString("v1v2group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_SNMPv2c, + new OctetString("cpublic"), + new OctetString("v1v2group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("SHADES"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("MD5DES"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("TEST"), + new OctetString("v3test"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("SHA"), + new OctetString("v3restricted"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("SHAAES128"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("SHAAES192"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("SHAAES256"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("MD5AES128"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("MD5AES192"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("MD5AES256"), + new OctetString("v3group"), + StorageType.nonVolatile); + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("aboba"), + new OctetString("v3group"), + StorageType.nonVolatile); + //============================================// + // agent5-auth-priv + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("agent5"), + new OctetString("v3group"), + StorageType.nonVolatile); + //===========================================// + // agent002 + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("agent002"), + new OctetString("v3group"), + StorageType.nonVolatile); + //===========================================// + // user001-auth-no-priv + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("user001"), + new OctetString("group001"), + StorageType.nonVolatile); + //===========================================// + + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("v3notify"), + new OctetString("v3group"), + StorageType.nonVolatile); + + //===========================================// + // group auth no priv + vacm.addGroup(SecurityModel.SECURITY_MODEL_USM, + new OctetString("v3notify-auth"), + new OctetString("group001"), + StorageType.nonVolatile); + //===========================================// + + + + // my conf + vacm.addAccess(new OctetString("group001"), new OctetString("public"), + SecurityModel.SECURITY_MODEL_USM, + SecurityLevel.AUTH_NOPRIV, + MutableVACM.VACM_MATCH_EXACT, + new OctetString("fullReadView"), + new OctetString("fullWriteView"), + new OctetString("fullNotifyView"), + StorageType.nonVolatile); + + vacm.addAccess(new OctetString("v1v2group"), new OctetString("public"), + SecurityModel.SECURITY_MODEL_ANY, + SecurityLevel.NOAUTH_NOPRIV, + MutableVACM.VACM_MATCH_EXACT, + new OctetString("fullReadView"), + new OctetString("fullWriteView"), + new OctetString("fullNotifyView"), + StorageType.nonVolatile); + vacm.addAccess(new OctetString("v3group"), new OctetString(), + SecurityModel.SECURITY_MODEL_USM, + SecurityLevel.AUTH_PRIV, + MutableVACM.VACM_MATCH_EXACT, + new OctetString("fullReadView"), + new OctetString("fullWriteView"), + new OctetString("fullNotifyView"), + StorageType.nonVolatile); + vacm.addAccess(new OctetString("v3restricted"), new OctetString(), + SecurityModel.SECURITY_MODEL_USM, + SecurityLevel.NOAUTH_NOPRIV, + MutableVACM.VACM_MATCH_EXACT, + new OctetString("restrictedReadView"), + new OctetString("restrictedWriteView"), + new OctetString("restrictedNotifyView"), + StorageType.nonVolatile); + vacm.addAccess(new OctetString("v3test"), new OctetString(), + SecurityModel.SECURITY_MODEL_USM, + SecurityLevel.AUTH_PRIV, + MutableVACM.VACM_MATCH_EXACT, + new OctetString("testReadView"), + new OctetString("testWriteView"), + new OctetString("testNotifyView"), + StorageType.nonVolatile); + + vacm.addViewTreeFamily(new OctetString("fullReadView"), new OID("1.3"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("fullWriteView"), new OID("1.3"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("fullNotifyView"), new OID("1.3"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + + vacm.addViewTreeFamily(new OctetString("restrictedReadView"), + new OID("1.3.6.1.2"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("restrictedWriteView"), + new OID("1.3.6.1.2.1"), + new OctetString(), + VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("restrictedNotifyView"), + new OID("1.3.6.1.2"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("restrictedNotifyView"), + new OID("1.3.6.1.6.3.1"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); - USM usm = new USM(); - SecurityModels.getInstance().addSecurityModel(usm); + vacm.addViewTreeFamily(new OctetString("testReadView"), + new OID("1.3.6.1.2"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("testReadView"), + new OID("1.3.6.1.2.1.1"), + new OctetString(), VacmMIB.vacmViewExcluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("testWriteView"), + new OID("1.3.6.1.2.1"), + new OctetString(), + VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + vacm.addViewTreeFamily(new OctetString("testNotifyView"), + new OID("1.3.6.1.2"), + new OctetString(), VacmMIB.vacmViewIncluded, + StorageType.nonVolatile); + } + + protected void addUsmUser(USM usm) { + UsmUser user = new UsmUser(new OctetString("SHADES"), + AuthSHA.ID, + new OctetString("SHADESAuthPassword"), + PrivDES.ID, + new OctetString("SHADESPrivPassword")); +// usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + usm.addUser(user.getSecurityName(), null, user); + user = new UsmUser(new OctetString("TEST"), + AuthSHA.ID, + new OctetString("maplesyrup"), + PrivDES.ID, + new OctetString("maplesyrup")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("SHA"), + AuthSHA.ID, + new OctetString("SHAAuthPassword"), + null, + null); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("SHADES"), + AuthSHA.ID, + new OctetString("SHADESAuthPassword"), + PrivDES.ID, + new OctetString("SHADESPrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("MD5DES"), + AuthMD5.ID, + new OctetString("MD5DESAuthPassword"), + PrivDES.ID, + new OctetString("MD5DESPrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("SHAAES128"), + AuthSHA.ID, + new OctetString("SHAAES128AuthPassword"), + PrivAES128.ID, + new OctetString("SHAAES128PrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("SHAAES192"), + AuthSHA.ID, + new OctetString("SHAAES192AuthPassword"), + PrivAES192.ID, + new OctetString("SHAAES192PrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("SHAAES256"), + AuthSHA.ID, + new OctetString("SHAAES256AuthPassword"), + PrivAES256.ID, + new OctetString("SHAAES256PrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + + user = new UsmUser(new OctetString("MD5AES128"), + AuthMD5.ID, + new OctetString("MD5AES128AuthPassword"), + PrivAES128.ID, + new OctetString("MD5AES128PrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("MD5AES192"), + AuthHMAC192SHA256.ID, + new OctetString("MD5AES192AuthPassword"), + PrivAES192.ID, + new OctetString("MD5AES192PrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + //============================================================== + user = new UsmUser(new OctetString("MD5AES256"), + AuthMD5.ID, + new OctetString("MD5AES256AuthPassword"), + PrivAES256.ID, + new OctetString("MD5AES256PrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + user = new UsmUser(new OctetString("MD5AES256"), + AuthMD5.ID, + new OctetString("MD5AES256AuthPassword"), + PrivAES256.ID, + new OctetString("MD5AES256PrivPassword")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + + OctetString securityName = new OctetString("aboba"); + OctetString authenticationPassphrase = new OctetString("abobaaboba"); + OctetString privacyPassphrase = new OctetString("abobaaboba"); OID authenticationProtocol = AuthSHA.ID; - OID privacyProtocol = PrivDES.ID; + OID privacyProtocol = PrivDES.ID; // FIXME: to config + user = new UsmUser(securityName, authenticationProtocol, authenticationPassphrase, privacyProtocol, privacyPassphrase); + usm.addUser(user); + + //===============================================================// + user = new UsmUser(new OctetString("agent5"), + AuthSHA.ID, + new OctetString("authpass"), + PrivDES.ID, + new OctetString("privpass")); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + //===============================================================// + // user001 + user = new UsmUser(new OctetString("user001"), + AuthSHA.ID, + new OctetString("authpass"), + null, null); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + //===============================================================// + // user002 + user = new UsmUser(new OctetString("user001"), + null, + null, + null, null); + usm.addUser(user.getSecurityName(), usm.getLocalEngineID(), user); + //===============================================================// + + user = new UsmUser(new OctetString("v3notify"), + null, + null, + null, + null); + usm.addUser(user.getSecurityName(), null, user); + + this.usm = usm; + } + + private static DefaultMOTable createStaticIfXTable() { + MOTableSubIndex[] subIndexes = + new MOTableSubIndex[] { new MOTableSubIndex(SMIConstants.SYNTAX_INTEGER) }; + MOTableIndex indexDef = new MOTableIndex(subIndexes, false); + MOColumn[] columns = new MOColumn[19]; + int c = 0; + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_OCTET_STRING, + MOAccessImpl.ACCESS_READ_ONLY); // ifName + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifInMulticastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifInBroadcastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifOutMulticastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifOutBroadcastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCInOctets + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCInUcastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCInMulticastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCInBroadcastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCOutOctets + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCOutUcastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCOutMulticastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_COUNTER32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHCOutBroadcastPkts + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_INTEGER, + MOAccessImpl.ACCESS_READ_WRITE); // ifLinkUpDownTrapEnable + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_GAUGE32, + MOAccessImpl.ACCESS_READ_ONLY); // ifHighSpeed + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_INTEGER, + MOAccessImpl.ACCESS_READ_WRITE); // ifPromiscuousMode + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_INTEGER, + MOAccessImpl.ACCESS_READ_ONLY); // ifConnectorPresent + columns[c++] = + new MOMutableColumn(c, SMIConstants.SYNTAX_OCTET_STRING, // ifAlias + MOAccessImpl.ACCESS_READ_WRITE, null); + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_TIMETICKS, + MOAccessImpl.ACCESS_READ_ONLY); // ifCounterDiscontinuityTime + + DefaultMOTable ifXTable = + new DefaultMOTable(new OID("1.3.6.1.2.1.31.1.1.1"), indexDef, columns); + MOMutableTableModel model = (MOMutableTableModel) ifXTable.getModel(); + Variable[] rowValues1 = new Variable[] { + new OctetString("Ethernet-0"), + new Integer32(1), + new Integer32(2), + new Integer32(3), + new Integer32(4), + new Integer32(5), + new Integer32(6), + new Integer32(7), + new Integer32(8), + new Integer32(9), + new Integer32(10), + new Integer32(11), + new Integer32(12), + new Integer32(13), + new Integer32(14), + new Integer32(15), + new Integer32(16), + new OctetString("My eth"), + new TimeTicks(1000) + }; + Variable[] rowValues2 = new Variable[] { + new OctetString("Loopback"), + new Integer32(21), + new Integer32(22), + new Integer32(23), + new Integer32(24), + new Integer32(25), + new Integer32(26), + new Integer32(27), + new Integer32(28), + new Integer32(29), + new Integer32(30), + new Integer32(31), + new Integer32(32), + new Integer32(33), + new Integer32(34), + new Integer32(35), + new Integer32(36), + new OctetString("My loop"), + new TimeTicks(2000) + }; + model.addRow(new DefaultMOMutableRow2PC(new OID("1"), rowValues1)); + model.addRow(new DefaultMOMutableRow2PC(new OID("2"), rowValues2)); + ifXTable.setVolatile(true); + return ifXTable; + } + + private static DefaultMOTable createStaticIfTable() { + MOTableSubIndex[] subIndexes = + new MOTableSubIndex[] { new MOTableSubIndex(SMIConstants.SYNTAX_INTEGER) }; + MOTableIndex indexDef = new MOTableIndex(subIndexes, false); + MOColumn[] columns = new MOColumn[8]; + int c = 0; + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_INTEGER, + MOAccessImpl.ACCESS_READ_ONLY); // ifIndex + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_OCTET_STRING, + MOAccessImpl.ACCESS_READ_ONLY); // ifDescr + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_INTEGER, + MOAccessImpl.ACCESS_READ_ONLY); // ifType + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_INTEGER, + MOAccessImpl.ACCESS_READ_ONLY); // ifMtu + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_GAUGE32, + MOAccessImpl.ACCESS_READ_ONLY); // ifSpeed + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_OCTET_STRING, + MOAccessImpl.ACCESS_READ_ONLY); // ifPhysAddress + columns[c++] = + new MOMutableColumn(c, SMIConstants.SYNTAX_INTEGER, // ifAdminStatus + MOAccessImpl.ACCESS_READ_WRITE, null); + columns[c++] = + new MOColumn(c, SMIConstants.SYNTAX_INTEGER, + MOAccessImpl.ACCESS_READ_ONLY); // ifOperStatus - UsmUser user = new UsmUser(new OctetString(securityName), authenticationProtocol, new OctetString(authenticationPassphrase), privacyProtocol, new OctetString(privacyPassphrase)); + DefaultMOTable ifTable = + new DefaultMOTable(new OID("1.3.6.1.2.1.2.2.1"), indexDef, columns); + MOMutableTableModel model = (MOMutableTableModel) ifTable.getModel(); + Variable[] rowValues1 = new Variable[] { + new Integer32(1), + new OctetString("eth0"), + new Integer32(6), + new Integer32(1500), + new Gauge32(100000000), + new OctetString("00:00:00:00:01"), + new Integer32(1), + new Integer32(1) + }; + Variable[] rowValues2 = new Variable[] { + new Integer32(2), + new OctetString("loopback"), + new Integer32(24), + new Integer32(1500), + new Gauge32(10000000), + new OctetString("00:00:00:00:02"), + new Integer32(1), + new Integer32(1) + }; + model.addRow(new DefaultMOMutableRow2PC(new OID("1"), rowValues1)); + model.addRow(new DefaultMOMutableRow2PC(new OID("2"), rowValues2)); + ifTable.setVolatile(true); + return ifTable; } - public void initV3(UsmUser user, String securityName) { -// snmp.getUSM().addUser(user); + private static DefaultMOTable createStaticSnmp4sTable() { + MOTableSubIndex[] subIndexes = + new MOTableSubIndex[] { new MOTableSubIndex(SMIConstants.SYNTAX_INTEGER) }; + MOTableIndex indexDef = new MOTableIndex(subIndexes, false); + MOColumn[] columns = new MOColumn[8]; + int c = 0; + columns[c++] = new MOColumn(c, SMIConstants.SYNTAX_NULL, MOAccessImpl.ACCESS_READ_ONLY); // testNull + columns[c++] = new MOColumn(c, SMIConstants.SYNTAX_INTEGER, MOAccessImpl.ACCESS_READ_ONLY); // testBoolean + columns[c++] = new MOColumn(c, SMIConstants.SYNTAX_INTEGER, MOAccessImpl.ACCESS_READ_ONLY); // ifType + columns[c++] = new MOColumn(c, SMIConstants.SYNTAX_INTEGER, MOAccessImpl.ACCESS_READ_ONLY); // ifMtu + columns[c++] = new MOColumn(c, SMIConstants.SYNTAX_GAUGE32, MOAccessImpl.ACCESS_READ_ONLY); // ifSpeed + columns[c++] = new MOColumn(c, SMIConstants.SYNTAX_OCTET_STRING, MOAccessImpl.ACCESS_READ_ONLY); //ifPhysAddress + columns[c++] = new MOMutableColumn(c, SMIConstants.SYNTAX_INTEGER, MOAccessImpl.ACCESS_READ_WRITE, + null); + // ifAdminStatus + columns[c++] = new MOColumn(c, SMIConstants.SYNTAX_INTEGER, MOAccessImpl.ACCESS_READ_ONLY); + // ifOperStatus + + DefaultMOTable ifTable = + new DefaultMOTable(new OID("1.3.6.1.4.1.50000.1.1"), indexDef, columns); + MOMutableTableModel model = (MOMutableTableModel) ifTable.getModel(); + Variable[] rowValues1 = new Variable[] { + new Integer32(1), + new OctetString("eth0"), + new Integer32(6), + new Integer32(1500), + new Gauge32(100000000), + new OctetString("00:00:00:00:01"), + new Integer32(1), + new Integer32(1) + }; + Variable[] rowValues2 = new Variable[] { + new Integer32(2), + new OctetString("loopback"), + new Integer32(24), + new Integer32(1500), + new Gauge32(10000000), + new OctetString("00:00:00:00:02"), + new Integer32(1), + new Integer32(1) + }; + model.addRow(new DefaultMOMutableRow2PC(new OID("1"), rowValues1)); + model.addRow(new DefaultMOMutableRow2PC(new OID("2"), rowValues2)); + ifTable.setVolatile(true); + return ifTable; + } + + protected void initTransportMappings() throws IOException { + transportMappings = new TransportMapping[2]; + Address addr = GenericAddress.parse(address); + TransportMapping tm = + TransportMappings.getInstance().createTransportMapping(addr); + transportMappings[0] = tm; + transportMappings[1] = new DefaultTcpTransportMapping(new TcpAddress(address)); + } + + public void start(String ip, String port) throws IOException { + address = ip + "/" + port; + //BasicConfigurator.configure(); + init(); + addShutdownHook(); +// loadConfig(ImportModes.REPLACE_CREATE); + getServer().addContext(new OctetString("public")); + finishInit(); + run(); + sendColdStartNotification(); + } + + protected void unregisterManagedObjects() { + // here we should unregister those objects previously registered... + } + + protected void addCommunities(SnmpCommunityMIB communityMIB) { + Variable[] com2sec = new Variable[] { + new OctetString("public"), // community name + new OctetString("cpublic"), // security name + getAgent().getContextEngineID(), // local engine ID + new OctetString("public"), // default context name + new OctetString(), // transport tag + new Integer32(StorageType.nonVolatile), // storage type + new Integer32(RowStatus.active) // row status + }; + MOTableRow row = + communityMIB.getSnmpCommunityEntry().createRow( + new OctetString("public2public").toSubIndex(true), com2sec); + communityMIB.getSnmpCommunityEntry().addRow((SnmpCommunityMIB.SnmpCommunityEntryRow) row); +// snmpCommunityMIB.setSourceAddressFiltering(true); + } + + protected void registerSnmpMIBs() { + heartbeatMIB = new Snmp4jHeartbeatMib(super.getNotificationOriginator(), + new OctetString(), + super.snmpv2MIB.getSysUpTime()); + agentppSimulationMIB = new AgentppSimulationMib(); + super.registerSnmpMIBs(); + } - UserTarget userTarget = new UserTarget(); - userTarget.setSecurityName(new OctetString(securityName)); - userTarget.setSecurityLevel(SecurityLevel.AUTH_PRIV); + protected void initMessageDispatcher() { + this.dispatcher = new MessageDispatcherImpl(); + this.mpv3 = new MPv3(this.agent.getContextEngineID().getValue()); + this.usm = new USM(SecurityProtocols.getInstance(), this.agent.getContextEngineID(), this.updateEngineBoots()); + SecurityModels.getInstance().addSecurityModel(this.usm); + SecurityProtocols.getInstance().addDefaultProtocols(); + this.dispatcher.addMessageProcessingModel(new MPv1()); + this.dispatcher.addMessageProcessingModel(new MPv2c()); + this.dispatcher.addMessageProcessingModel(this.mpv3); + this.initSnmpSession(); } -} +} \ No newline at end of file diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTest.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTestV2.java similarity index 98% rename from common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTest.java rename to common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTestV2.java index 29a8c085c8..417204296c 100644 --- a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTest.java +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTestV2.java @@ -19,7 +19,7 @@ import java.io.IOException; import java.util.Map; import java.util.Scanner; -public class SnmpTest { +public class SnmpTestV2 { public static void main(String[] args) throws IOException { SnmpDeviceSimulatorV2 device = new SnmpDeviceSimulatorV2(1610, "public"); diff --git a/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTestV3.java b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTestV3.java new file mode 100644 index 0000000000..8e40deb17d --- /dev/null +++ b/common/transport/snmp/src/test/java/org/thingsboard/server/transport/snmp/SnmpTestV3.java @@ -0,0 +1,46 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp; + +import org.snmp4j.CommandResponderEvent; +import org.snmp4j.agent.CommandProcessor; +import org.snmp4j.mp.MPv3; +import org.snmp4j.smi.OctetString; + +import java.io.IOException; +import java.util.Map; +import java.util.Scanner; + +public class SnmpTestV3 { + public static void main(String[] args) throws IOException { + SnmpDeviceSimulatorV3 device = new SnmpDeviceSimulatorV3(new CommandProcessor(new OctetString(MPv3.createLocalEngineID())) { + @Override + public void processPdu(CommandResponderEvent event) { + System.out.println("event: " + event); + } + }); + device.start("0.0.0.0", "1610"); + + device.setUpMappings(Map.of( + ".1.3.6.1.2.1.1.1.50", "12", + ".1.3.6.1.2.1.2.1.52", "56", + ".1.3.6.1.2.1.3.1.54", "yes", + ".1.3.6.1.2.1.7.1.58", "" + )); + + new Scanner(System.in).nextLine(); + } +} diff --git a/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json b/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json index f8291a090b..f74ebca0bf 100644 --- a/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json +++ b/common/transport/snmp/src/test/resources/snmp-device-profile-transport-config.json @@ -38,16 +38,6 @@ "dataType": "STRING" } ] - }, - { - "spec": "TELEMETRY_TRAPS_RECEIVING", - "mappings": [ - { - "oid": ".1.3.6.1.2.8.7.1.56", - "key": "temperature_trap", - "dataType": "LONG" - } - ] } ] } diff --git a/common/transport/snmp/src/test/resources/snmp-device-transport-config-v3.json b/common/transport/snmp/src/test/resources/snmp-device-transport-config-v3.json new file mode 100644 index 0000000000..039e03fa53 --- /dev/null +++ b/common/transport/snmp/src/test/resources/snmp-device-transport-config-v3.json @@ -0,0 +1,13 @@ +{ + "address": "192.168.3.23", + "port": 1610, + "protocolVersion": "V3", + + "username": "tb-user", + "engineId": "qwertyuioa", + "securityName": "tb-user", + "authenticationProtocol": "SHA_512", + "authenticationPassphrase": "sdfghjkloifgh", + "privacyProtocol": "DES", + "privacyPassphrase": "rtytguijokod" +} \ No newline at end of file diff --git a/common/transport/snmp/src/test/resources/snmp-device-transport-config.json b/common/transport/snmp/src/test/resources/snmp-device-transport-config.json index 9a97b75e40..c73d817bfb 100644 --- a/common/transport/snmp/src/test/resources/snmp-device-transport-config.json +++ b/common/transport/snmp/src/test/resources/snmp-device-transport-config.json @@ -1,6 +1,6 @@ { "address": "127.0.0.1", "port": 1610, - "password": "public", + "community": "public", "protocolVersion": "V2C" } \ No newline at end of file From 415bf570bac6b422f880622163924464d6ca390b Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 7 Apr 2021 19:22:28 +0300 Subject: [PATCH 11/32] Refactor --- .../SnmpDeviceTransportConfiguration.java | 26 +++++++-- ...mpDeviceProfileTransportConfiguration.java | 18 +++++-- ...eatingQueryingSnmpCommunicationConfig.java | 2 +- ...ributesSettingSnmpCommunicationConfig.java | 5 -- .../snmp/configs/SnmpCommunicationConfig.java | 8 +-- .../transport/snmp/SnmpAuthService.java | 3 +- .../transport/snmp/SnmpTransportContext.java | 54 ++++++++++--------- .../snmp/service/SnmpTransportService.java | 27 ++++++++-- .../snmp/session/DeviceSessionContext.java | 14 ++--- 9 files changed, 98 insertions(+), 59 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index 24ab463129..68a9d1f218 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -17,16 +17,21 @@ package org.thingsboard.server.common.data.device.data; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; +import lombok.ToString; +import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.transport.snmp.AuthenticationProtocol; import org.thingsboard.server.common.data.transport.snmp.PrivacyProtocol; import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; +import java.util.Objects; + @Data +@ToString(of = {"host", "port", "protocolVersion"}) public class SnmpDeviceTransportConfiguration implements DeviceTransportConfiguration { - private String address; - private int port; + private String host; + private Integer port; private SnmpProtocolVersion protocolVersion; /* @@ -60,6 +65,21 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur @JsonIgnore private boolean isValid() { - return true; + boolean isValid = StringUtils.isNotBlank(host) && port != null && protocolVersion != null; + if (isValid) { + switch (protocolVersion) { + case V1: + case V2C: + isValid = StringUtils.isNotEmpty(community); + break; + case V3: + isValid = StringUtils.isNotBlank(username) && StringUtils.isNotBlank(securityName) + && contextName != null && authenticationProtocol != null + && StringUtils.isNotBlank(authenticationPassphrase) + && privacyProtocol != null && privacyPassphrase != null && engineId != null; + break; + } + } + return isValid; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java index 843a51304e..5d6afc7155 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java @@ -17,15 +17,21 @@ package org.thingsboard.server.common.data.device.profile; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; +import org.apache.commons.lang3.ArrayUtils; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; import org.thingsboard.server.common.data.transport.snmp.configs.SnmpCommunicationConfig; +import java.util.Collections; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Data public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTransportConfiguration { - private int timeoutMs; - private int retries; + private Integer timeoutMs; + private Integer retries; private List communicationConfigs; @Override @@ -36,12 +42,16 @@ public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTra @Override public void validate() { if (!isValid()) { - throw new IllegalArgumentException("Transport configuration is not valid"); + throw new IllegalArgumentException("SNMP transport configuration is not valid"); } } @JsonIgnore private boolean isValid() { - return true; + return timeoutMs != null && timeoutMs >= 0 && retries != null && retries >= 0 + && communicationConfigs != null && !communicationConfigs.isEmpty() + && communicationConfigs.stream().allMatch(config -> config != null && config.isValid()) + && communicationConfigs.stream().flatMap(config -> config.getMappings().stream()).map(SnmpMapping::getOid) + .distinct().count() == communicationConfigs.stream().mapToInt(config -> config.getMappings().size()).sum(); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java index 764bf97b15..19a009457a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java @@ -31,6 +31,6 @@ public abstract class RepeatingQueryingSnmpCommunicationConfig extends SnmpCommu @Override public boolean isValid() { - return true; + return super.isValid() && queryingFrequencyMs != null && queryingFrequencyMs > 0; } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java index 1c63dae618..74c67f957a 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java @@ -28,9 +28,4 @@ public class SharedAttributesSettingSnmpCommunicationConfig extends SnmpCommunic public SnmpMethod getMethod() { return SnmpMethod.SET; } - - @Override - public boolean isValid() { - return true; - } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java index 7e1ee27237..acc0df0dd7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java @@ -49,12 +49,6 @@ public abstract class SnmpCommunicationConfig { @JsonIgnore public boolean isValid() { - return true; - } - - public void validate() { - if (!isValid()) { - throw new IllegalArgumentException("Communication config is not valid"); - } + return mappings != null && !mappings.isEmpty() && mappings.stream().allMatch(mapping -> mapping != null && mapping.isValid()); } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java index 59b31d593c..81187fb33e 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java @@ -24,7 +24,6 @@ import org.snmp4j.security.SecurityLevel; import org.snmp4j.security.SecurityModel; import org.snmp4j.security.SecurityProtocols; import org.snmp4j.security.USM; -import org.snmp4j.security.UsmUser; import org.snmp4j.smi.GenericAddress; import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; @@ -98,7 +97,7 @@ public class SnmpAuthService { throw new UnsupportedOperationException("SNMP protocol version " + protocolVersion + " is not supported"); } - target.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getAddress() + "/" + deviceTransportConfig.getPort())); + target.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getHost() + "/" + deviceTransportConfig.getPort())); target.setTimeout(profileTransportConfig.getTimeoutMs()); target.setRetries(profileTransportConfig.getRetries()); target.setVersion(protocolVersion.getCode()); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 2b0ce984aa..d7441c7c1d 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -89,18 +89,12 @@ public class SnmpTransportContext extends TransportContext { managedDevicesIds.stream() .map(protoEntityService::getDeviceById) .collect(Collectors.toList()) - .forEach(device -> { - try { - establishDeviceSession(device); - } catch (Exception e) { - log.error("Failed to establish session for SNMP device {}: {}", device.getId(), e.getMessage()); - } - }); + .forEach(this::establishDeviceSession); } private void establishDeviceSession(Device device) { if (device == null) return; - log.info("Establishing SNMP device session for device {}", device.getId()); + log.info("Establishing SNMP session for device {}", device.getId()); DeviceProfileId deviceProfileId = device.getDeviceProfileId(); DeviceProfile deviceProfile = deviceProfileCache.get(deviceProfileId); @@ -114,18 +108,24 @@ public class SnmpTransportContext extends TransportContext { SnmpDeviceProfileTransportConfiguration profileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); SnmpDeviceTransportConfiguration deviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - DeviceSessionContext deviceSessionContext = new DeviceSessionContext( - device, deviceProfile, credentials.getCredentialsId(), - profileTransportConfiguration, deviceTransportConfiguration, this - ); - registerSessionMsgListener(deviceSessionContext); + DeviceSessionContext deviceSessionContext; + try { + deviceSessionContext = new DeviceSessionContext( + device, deviceProfile, credentials.getCredentialsId(), + profileTransportConfiguration, deviceTransportConfiguration, this + ); + registerSessionMsgListener(deviceSessionContext); + } catch (Exception e) { + log.error("Failed to establish session for SNMP device {}: {}", device.getId(), e.getMessage()); + return; + } sessions.put(device.getId(), deviceSessionContext); snmpTransportService.createQueryingTasks(deviceSessionContext); log.info("Established SNMP device session for device {}", device.getId()); } private void updateDeviceSession(DeviceSessionContext sessionContext, Device device, DeviceProfile deviceProfile) { - log.info("Updating SNMP device session for device {}", device.getId()); + log.info("Updating SNMP session for device {}", device.getId()); DeviceCredentials credentials = protoEntityService.getDeviceCredentialsByDeviceId(device.getId()); if (credentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { @@ -137,16 +137,20 @@ public class SnmpTransportContext extends TransportContext { SnmpDeviceProfileTransportConfiguration newProfileTransportConfiguration = (SnmpDeviceProfileTransportConfiguration) deviceProfile.getProfileData().getTransportConfiguration(); SnmpDeviceTransportConfiguration newDeviceTransportConfiguration = (SnmpDeviceTransportConfiguration) device.getDeviceData().getTransportConfiguration(); - if (!newProfileTransportConfiguration.equals(sessionContext.getProfileTransportConfiguration())) { - sessionContext.setProfileTransportConfiguration(newProfileTransportConfiguration); - sessionContext.initializeTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); - snmpTransportService.cancelQueryingTasks(sessionContext); - snmpTransportService.createQueryingTasks(sessionContext); - } else if (!newDeviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { - sessionContext.setDeviceTransportConfiguration(newDeviceTransportConfiguration); - sessionContext.initializeTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); - } else { - log.trace("Configuration of the device {} was not updated", device); + try { + if (!newProfileTransportConfiguration.equals(sessionContext.getProfileTransportConfiguration())) { + sessionContext.setProfileTransportConfiguration(newProfileTransportConfiguration); + sessionContext.initializeTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); + snmpTransportService.cancelQueryingTasks(sessionContext); + snmpTransportService.createQueryingTasks(sessionContext); + } else if (!newDeviceTransportConfiguration.equals(sessionContext.getDeviceTransportConfiguration())) { + sessionContext.setDeviceTransportConfiguration(newDeviceTransportConfiguration); + sessionContext.initializeTarget(newProfileTransportConfiguration, newDeviceTransportConfiguration); + } else { + log.trace("Configuration of the device {} was not updated", device); + } + } catch (Exception e) { + log.error("Failed to update session for SNMP device {}: {}", sessionContext.getDeviceId(), e.getMessage()); } } @@ -156,8 +160,8 @@ public class SnmpTransportContext extends TransportContext { sessionContext.close(); snmpAuthService.cleanUpSnmpAuthInfo(sessionContext); transportService.deregisterSession(sessionContext.getSessionInfo()); - sessions.remove(sessionContext.getDeviceId()); snmpTransportService.cancelQueryingTasks(sessionContext); + sessions.remove(sessionContext.getDeviceId()); log.trace("Unregistered and removed session"); } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 2f26bfa98c..94a18f3d54 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -28,9 +28,11 @@ import org.snmp4j.mp.MPv3; import org.snmp4j.security.SecurityModels; import org.snmp4j.security.SecurityProtocols; import org.snmp4j.security.USM; +import org.snmp4j.smi.Integer32; import org.snmp4j.smi.Null; import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.Variable; import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultTcpTransportMapping; import org.snmp4j.transport.DefaultUdpTransportMapping; @@ -143,7 +145,7 @@ public class SnmpTransportService implements TbTransportService { sendRequest(sessionContext, communicationConfig); } } catch (Exception e) { - log.error("Failed to send SNMP request for device {}: {}", sessionContext.getDeviceId(), e.getMessage()); + log.error("Failed to send SNMP request for device {}: {}", sessionContext.getDeviceId(), e.toString()); } }, queryingFrequency, queryingFrequency, TimeUnit.MILLISECONDS); } @@ -192,7 +194,24 @@ public class SnmpTransportService implements TbTransportService { pdu.addAll(communicationConfig.getMappings().stream() .filter(mapping -> values.isEmpty() || values.containsKey(mapping.getKey())) .map(mapping -> Optional.ofNullable(values.get(mapping.getKey())) - .map(value -> new VariableBinding(new OID(mapping.getOid()), new OctetString(values.get(mapping.getKey())))) + .map(value -> { + Variable variable; + switch (mapping.getDataType()) { + case LONG: + try { + variable = new Integer32(Integer.parseInt(value)); + break; + } catch (NumberFormatException ignored) { + } + case DOUBLE: + case BOOLEAN: + case STRING: + case JSON: + default: + variable = new OctetString(value); + } + return new VariableBinding(new OID(mapping.getOid()), variable); + }) .orElseGet(() -> new VariableBinding(new OID(mapping.getOid())))) .collect(Collectors.toList())); @@ -267,7 +286,9 @@ public class SnmpTransportService implements TbTransportService { responses.forEach((spec, response) -> { Optional.ofNullable(responseProcessors.get(spec)) .ifPresent(responseProcessor -> { - responseProcessor.accept(response, sessionContext); + if (!response.entrySet().isEmpty()) { + responseProcessor.accept(response, sessionContext); + } }); }); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index c5a329547f..b3a38545a8 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -63,8 +63,6 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S private final Device device; private final SnmpTransportContext snmpTransportContext; - private final SnmpTransportService snmpTransportService; - private final SnmpAuthService snmpAuthService; @Getter @Setter @@ -80,7 +78,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S public DeviceSessionContext(Device device, DeviceProfile deviceProfile, String token, SnmpDeviceProfileTransportConfiguration profileTransportConfiguration, SnmpDeviceTransportConfiguration deviceTransportConfiguration, - SnmpTransportContext snmpTransportContext) { + SnmpTransportContext snmpTransportContext) throws Exception { super(UUID.randomUUID()); super.setDeviceId(device.getId()); super.setDeviceProfile(deviceProfile); @@ -88,8 +86,6 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S this.token = token; this.snmpTransportContext = snmpTransportContext; - this.snmpTransportService = snmpTransportContext.getSnmpTransportService(); - this.snmpAuthService = snmpTransportContext.getSnmpAuthService(); this.profileTransportConfiguration = profileTransportConfiguration; this.deviceTransportConfiguration = deviceTransportConfiguration; @@ -113,13 +109,13 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onResponse(ResponseEvent event) { if (isActive) { - snmpTransportService.processResponseEvent(this, event); + snmpTransportContext.getSnmpTransportService().processResponseEvent(this, event); } } - public void initializeTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) { + public void initializeTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) throws Exception { log.trace("Initializing target for SNMP session of device {}", device); - this.target = snmpAuthService.setUpSnmpTarget(profileTransportConfig, deviceTransportConfig); + this.target = snmpTransportContext.getSnmpAuthService().setUpSnmpTarget(profileTransportConfig, deviceTransportConfig); log.info("SNMP target initialized: {}", target); } @@ -152,7 +148,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S entry -> entry.getValue().isJsonPrimitive() ? entry.getValue().getAsString() : entry.getValue().toString() )); try { - snmpTransportService.sendRequest(this, communicationConfig, sharedAttributes); + snmpTransportContext.getSnmpTransportService().sendRequest(this, communicationConfig, sharedAttributes); } catch (Exception e) { log.error("Failed to send request with shared attributes to SNMP device {}: {}", getDeviceId(), e.getMessage()); } From 783b959577be49cbf020f379e09a0043dfcd95a9 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 14 Apr 2021 15:51:13 +0300 Subject: [PATCH 12/32] Refactor --- ...mpDeviceProfileTransportConfiguration.java | 11 +- .../transport/snmp/SnmpCommunicationSpec.java | 21 +- .../data/transport/snmp/SnmpMapping.java | 7 +- ...ltipleMappingsSnmpCommunicationConfig.java | 36 +++ ...eatingQueryingSnmpCommunicationConfig.java | 6 +- .../SnmpCommunicationConfig.java | 24 +- ...ibutesQueryingSnmpCommunicationConfig.java | 5 +- ...ributesSettingSnmpCommunicationConfig.java | 7 +- ...emetryQueryingSnmpCommunicationConfig.java | 5 +- ...CommandSettingSnmpCommunicationConfig.java | 59 +++++ ...sponseQueryingSnmpCommunicationConfig.java | 60 +++++ .../transport/snmp/SnmpTransportContext.java | 7 +- .../transport/snmp/service/PduMapper.java | 150 +++++++++++ .../snmp/{ => service}/SnmpAuthService.java | 8 +- .../snmp/service/SnmpTransportService.java | 244 ++++++------------ .../snmp/session/DeviceSessionContext.java | 38 +-- 16 files changed, 454 insertions(+), 234 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/MultipleMappingsSnmpCommunicationConfig.java rename common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/{configs => config}/RepeatingQueryingSnmpCommunicationConfig.java (88%) rename common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/{configs => config}/SnmpCommunicationConfig.java (75%) rename common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/{configs => config/impl}/ClientAttributesQueryingSnmpCommunicationConfig.java (89%) rename common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/{configs => config/impl}/SharedAttributesSettingSnmpCommunicationConfig.java (87%) rename common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/{configs => config/impl}/TelemetryQueryingSnmpCommunicationConfig.java (90%) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java create mode 100644 common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduMapper.java rename common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/{ => service}/SnmpAuthService.java (94%) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java index 5d6afc7155..bbf074eeda 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java @@ -17,16 +17,11 @@ package org.thingsboard.server.common.data.device.profile; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import org.apache.commons.lang3.ArrayUtils; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; -import org.thingsboard.server.common.data.transport.snmp.configs.SnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; -import java.util.Collections; import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.Stream; @Data public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTransportConfiguration { @@ -51,7 +46,7 @@ public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTra return timeoutMs != null && timeoutMs >= 0 && retries != null && retries >= 0 && communicationConfigs != null && !communicationConfigs.isEmpty() && communicationConfigs.stream().allMatch(config -> config != null && config.isValid()) - && communicationConfigs.stream().flatMap(config -> config.getMappings().stream()).map(SnmpMapping::getOid) - .distinct().count() == communicationConfigs.stream().mapToInt(config -> config.getMappings().size()).sum(); + && communicationConfigs.stream().flatMap(config -> config.getAllMappings().stream()).map(SnmpMapping::getOid) + .distinct().count() == communicationConfigs.stream().mapToInt(config -> config.getAllMappings().size()).sum(); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java index a2edf75968..a6643ecf1e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java @@ -16,22 +16,11 @@ package org.thingsboard.server.common.data.transport.snmp; public enum SnmpCommunicationSpec { - TELEMETRY_QUERYING(true), - CLIENT_ATTRIBUTES_QUERYING(true), + TELEMETRY_QUERYING, - SHARED_ATTRIBUTES_SETTING; + CLIENT_ATTRIBUTES_QUERYING, + SHARED_ATTRIBUTES_SETTING, - private final boolean isRepeatingQuerying; - - SnmpCommunicationSpec() { - this.isRepeatingQuerying = false; - } - - SnmpCommunicationSpec(boolean isRepeatingQuerying) { - this.isRepeatingQuerying = isRepeatingQuerying; - } - - public boolean isRepeatingQuerying() { - return isRepeatingQuerying; - } + TO_DEVICE_RPC_COMMAND_SETTING, + TO_DEVICE_RPC_RESPONSE_QUERYING } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java index 0269491461..d787f7446f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpMapping.java @@ -16,13 +16,17 @@ package org.thingsboard.server.common.data.transport.snmp; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.kv.DataType; import java.util.regex.Pattern; @Data +@AllArgsConstructor +@NoArgsConstructor public class SnmpMapping { private String oid; private String key; @@ -32,7 +36,6 @@ public class SnmpMapping { @JsonIgnore public boolean isValid() { - return StringUtils.isNotEmpty(oid) && OID_PATTERN.matcher(oid).matches() && - StringUtils.isNotBlank(key) && dataType != null; + return StringUtils.isNotEmpty(oid) && OID_PATTERN.matcher(oid).matches() && StringUtils.isNotBlank(key); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/MultipleMappingsSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/MultipleMappingsSnmpCommunicationConfig.java new file mode 100644 index 0000000000..f785c8788d --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/MultipleMappingsSnmpCommunicationConfig.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.config; + +import lombok.Data; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; + +import java.util.List; + +@Data +public abstract class MultipleMappingsSnmpCommunicationConfig implements SnmpCommunicationConfig { + protected List mappings; + + @Override + public boolean isValid() { + return mappings != null && !mappings.isEmpty() && mappings.stream().allMatch(mapping -> mapping != null && mapping.isValid()); + } + + @Override + public List getAllMappings() { + return mappings; + } +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/RepeatingQueryingSnmpCommunicationConfig.java similarity index 88% rename from common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/RepeatingQueryingSnmpCommunicationConfig.java index 19a009457a..38451d4142 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/RepeatingQueryingSnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/RepeatingQueryingSnmpCommunicationConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.transport.snmp.configs; +package org.thingsboard.server.common.data.transport.snmp.config; import lombok.Data; import lombok.EqualsAndHashCode; @@ -21,7 +21,7 @@ import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; @EqualsAndHashCode(callSuper = true) @Data -public abstract class RepeatingQueryingSnmpCommunicationConfig extends SnmpCommunicationConfig { +public abstract class RepeatingQueryingSnmpCommunicationConfig extends MultipleMappingsSnmpCommunicationConfig { private Long queryingFrequencyMs; @Override @@ -31,6 +31,6 @@ public abstract class RepeatingQueryingSnmpCommunicationConfig extends SnmpCommu @Override public boolean isValid() { - return super.isValid() && queryingFrequencyMs != null && queryingFrequencyMs > 0; + return queryingFrequencyMs != null && queryingFrequencyMs > 0 && super.isValid(); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/SnmpCommunicationConfig.java similarity index 75% rename from common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/SnmpCommunicationConfig.java index acc0df0dd7..e2ba29abfa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/SnmpCommunicationConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.transport.snmp.configs; +package org.thingsboard.server.common.data.transport.snmp.config; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -23,6 +23,9 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; +import org.thingsboard.server.common.data.transport.snmp.config.impl.ClientAttributesQueryingSnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.impl.SharedAttributesSettingSnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.impl.TelemetryQueryingSnmpCommunicationConfig; import java.util.List; @@ -33,22 +36,19 @@ import java.util.List; @Type(value = ClientAttributesQueryingSnmpCommunicationConfig.class, name = "CLIENT_ATTRIBUTES_QUERYING"), @Type(value = SharedAttributesSettingSnmpCommunicationConfig.class, name = "SHARED_ATTRIBUTES_SETTING") }) -public abstract class SnmpCommunicationConfig { - protected List mappings; +public interface SnmpCommunicationConfig { - public List getMappings() { - return mappings; - } - - public abstract SnmpCommunicationSpec getSpec(); + SnmpCommunicationSpec getSpec(); @JsonIgnore - public SnmpMethod getMethod() { + default SnmpMethod getMethod() { return null; } @JsonIgnore - public boolean isValid() { - return mappings != null && !mappings.isEmpty() && mappings.stream().allMatch(mapping -> mapping != null && mapping.isValid()); - } + List getAllMappings(); + + @JsonIgnore + boolean isValid(); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ClientAttributesQueryingSnmpCommunicationConfig.java similarity index 89% rename from common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesQueryingSnmpCommunicationConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ClientAttributesQueryingSnmpCommunicationConfig.java index 16198f8ef6..51bd2198e9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/ClientAttributesQueryingSnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ClientAttributesQueryingSnmpCommunicationConfig.java @@ -13,13 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.transport.snmp.configs; +package org.thingsboard.server.common.data.transport.snmp.config.impl; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.config.RepeatingQueryingSnmpCommunicationConfig; public class ClientAttributesQueryingSnmpCommunicationConfig extends RepeatingQueryingSnmpCommunicationConfig { + @Override public SnmpCommunicationSpec getSpec() { return SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/SharedAttributesSettingSnmpCommunicationConfig.java similarity index 87% rename from common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/SharedAttributesSettingSnmpCommunicationConfig.java index 74c67f957a..441395d279 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/SharedAttributesSettingSnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/SharedAttributesSettingSnmpCommunicationConfig.java @@ -13,12 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.transport.snmp.configs; +package org.thingsboard.server.common.data.transport.snmp.config.impl; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; +import org.thingsboard.server.common.data.transport.snmp.config.MultipleMappingsSnmpCommunicationConfig; + +public class SharedAttributesSettingSnmpCommunicationConfig extends MultipleMappingsSnmpCommunicationConfig { -public class SharedAttributesSettingSnmpCommunicationConfig extends SnmpCommunicationConfig { @Override public SnmpCommunicationSpec getSpec() { return SnmpCommunicationSpec.SHARED_ATTRIBUTES_SETTING; @@ -28,4 +30,5 @@ public class SharedAttributesSettingSnmpCommunicationConfig extends SnmpCommunic public SnmpMethod getMethod() { return SnmpMethod.SET; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/TelemetryQueryingSnmpCommunicationConfig.java similarity index 90% rename from common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryQueryingSnmpCommunicationConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/TelemetryQueryingSnmpCommunicationConfig.java index 68d3c81ed2..c3f82d64ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/configs/TelemetryQueryingSnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/TelemetryQueryingSnmpCommunicationConfig.java @@ -13,17 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.transport.snmp.configs; +package org.thingsboard.server.common.data.transport.snmp.config.impl; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.config.RepeatingQueryingSnmpCommunicationConfig; @EqualsAndHashCode(callSuper = true) @Data public class TelemetryQueryingSnmpCommunicationConfig extends RepeatingQueryingSnmpCommunicationConfig { + @Override public SnmpCommunicationSpec getSpec() { return SnmpCommunicationSpec.TELEMETRY_QUERYING; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..97f3de047b --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java @@ -0,0 +1,59 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.config.impl; + +import lombok.Data; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; +import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +@Data +public class ToDeviceRpcCommandSettingSnmpCommunicationConfig implements SnmpCommunicationConfig { + private SnmpMapping mapping; + + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.TO_DEVICE_RPC_COMMAND_SETTING; + } + + @Override + public SnmpMethod getMethod() { + return SnmpMethod.SET; + } + + public void setMapping(SnmpMapping mapping) { + this.mapping = mapping != null ? new SnmpMapping(mapping.getOid(), RPC_COMMAND_KEY_NAME, DataType.STRING) : null; + } + + @Override + public List getAllMappings() { + return Collections.singletonList(mapping); + } + + @Override + public boolean isValid() { + return mapping != null && mapping.isValid(); + } + + public static final String RPC_COMMAND_KEY_NAME = "rpcCommand"; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java new file mode 100644 index 0000000000..18bce94980 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.config.impl; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; +import org.thingsboard.server.common.data.transport.snmp.config.RepeatingQueryingSnmpCommunicationConfig; + +import java.util.Collections; +import java.util.List; + +@EqualsAndHashCode(callSuper = true) +@Data +public class ToDeviceRpcResponseQueryingSnmpCommunicationConfig extends RepeatingQueryingSnmpCommunicationConfig { + private SnmpMapping mapping; + + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.TO_DEVICE_RPC_RESPONSE_QUERYING; + } + + @Override + public SnmpMethod getMethod() { + return SnmpMethod.GET; + } + + public void setMapping(SnmpMapping mapping) { + this.mapping = mapping != null ? new SnmpMapping(mapping.getOid(), RPC_RESPONSE_KEY_NAME, DataType.STRING) : null; + } + + @Override + public List getAllMappings() { + return Collections.singletonList(mapping); + } + + @Override + public boolean isValid() { + return true; + } + + public static final String RPC_RESPONSE_KEY_NAME = "rpcResponse"; + +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index d7441c7c1d..d66d09ab0b 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -42,6 +42,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.SessionInfoProto; import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; import org.thingsboard.server.transport.snmp.service.ProtoTransportEntityService; +import org.thingsboard.server.transport.snmp.service.SnmpAuthService; import org.thingsboard.server.transport.snmp.service.SnmpTransportBalancingService; import org.thingsboard.server.transport.snmp.service.SnmpTransportService; import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; @@ -116,7 +117,7 @@ public class SnmpTransportContext extends TransportContext { ); registerSessionMsgListener(deviceSessionContext); } catch (Exception e) { - log.error("Failed to establish session for SNMP device {}: {}", device.getId(), e.getMessage()); + log.error("Failed to establish session for SNMP device {}: {}", device.getId(), e.toString()); return; } sessions.put(device.getId(), deviceSessionContext); @@ -177,7 +178,9 @@ public class SnmpTransportContext extends TransportContext { ); transportService.registerAsyncSession(sessionInfo, deviceSessionContext); - transportService.process(sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().build(), null); + transportService.process(sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().build(), TransportServiceCallback.EMPTY); + transportService.process(sessionInfo, TransportProtos.SubscribeToRPCMsg.newBuilder().build(), TransportServiceCallback.EMPTY); + deviceSessionContext.setSessionInfo(sessionInfo); deviceSessionContext.setDeviceInfo(msg.getDeviceInfo()); } else { diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduMapper.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduMapper.java new file mode 100644 index 0000000000..a3b5f77dfb --- /dev/null +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduMapper.java @@ -0,0 +1,150 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.snmp.service; + +import com.google.gson.JsonObject; +import lombok.extern.slf4j.Slf4j; +import org.snmp4j.PDU; +import org.snmp4j.ScopedPDU; +import org.snmp4j.smi.Integer32; +import org.snmp4j.smi.Null; +import org.snmp4j.smi.OID; +import org.snmp4j.smi.OctetString; +import org.snmp4j.smi.Variable; +import org.snmp4j.smi.VariableBinding; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; +import org.thingsboard.server.common.data.kv.DataType; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; +import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; +import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +@TbSnmpTransportComponent +@Service +@Slf4j +public class PduMapper { + public PDU createPdu(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) { + PDU pdu; + SnmpDeviceTransportConfiguration deviceTransportConfiguration = sessionContext.getDeviceTransportConfiguration(); + SnmpProtocolVersion snmpVersion = deviceTransportConfiguration.getProtocolVersion(); + switch (snmpVersion) { + case V1: + case V2C: + pdu = new PDU(); + break; + case V3: + ScopedPDU scopedPdu = new ScopedPDU(); + scopedPdu.setContextName(new OctetString(deviceTransportConfiguration.getContextName())); + scopedPdu.setContextEngineID(new OctetString(deviceTransportConfiguration.getEngineId())); + pdu = scopedPdu; + break; + default: + throw new UnsupportedOperationException("SNMP version " + snmpVersion + " is not supported"); + } + + pdu.setType(communicationConfig.getMethod().getCode()); + pdu.addAll(communicationConfig.getAllMappings().stream() + .filter(mapping -> values.isEmpty() || values.containsKey(mapping.getKey())) + .map(mapping -> Optional.ofNullable(values.get(mapping.getKey())) + .map(value -> { + Variable variable = toSnmpVariable(mapping, value); + return new VariableBinding(new OID(mapping.getOid()), variable); + }) + .orElseGet(() -> new VariableBinding(new OID(mapping.getOid())))) + .collect(Collectors.toList())); + + return pdu; + } + + private Variable toSnmpVariable(SnmpMapping mapping, String value) { + Variable variable; + switch (mapping.getDataType()) { + case LONG: + try { + variable = new Integer32(Integer.parseInt(value)); + break; + } catch (NumberFormatException ignored) { + } + case DOUBLE: + case BOOLEAN: + case STRING: + case JSON: + default: + variable = new OctetString(value); + } + return variable; + } + + + public JsonObject processPdu(PDU pdu, DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) { + List variablesBindings = IntStream.range(0, pdu.size()) + .mapToObj(pdu::get) + .filter(Objects::nonNull) + .filter(variableBinding -> !(variableBinding.getVariable() instanceof Null)) + .collect(Collectors.toList()); + JsonObject data = new JsonObject(); + + Map mappings = new HashMap<>(); + for (SnmpMapping mapping : communicationConfig.getAllMappings()) { + OID oid = new OID(mapping.getOid()); + mappings.put(oid, mapping); + } + + variablesBindings.forEach(variableBinding -> { + log.trace("Processing variable binding: {}", variableBinding); + + OID oid = variableBinding.getOid(); + SnmpMapping mapping = mappings.get(oid); + if (mapping == null) { + log.debug("No SNMP mapping for oid {}", oid); + return; + } + + processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), data); + }); + + return data; + } + + + private void processValue(String key, DataType dataType, String value, JsonObject result) { + switch (dataType) { + case LONG: + result.addProperty(key, Long.parseLong(value)); + break; + case BOOLEAN: + result.addProperty(key, Boolean.parseBoolean(value)); + break; + case DOUBLE: + result.addProperty(key, Double.parseDouble(value)); + break; + case STRING: + case JSON: + default: + result.addProperty(key, value); + } + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpAuthService.java similarity index 94% rename from common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java rename to common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpAuthService.java index 81187fb33e..c3ed188bb0 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpAuthService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpAuthService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.transport.snmp; +package org.thingsboard.server.transport.snmp.service; import lombok.RequiredArgsConstructor; import org.snmp4j.AbstractTarget; @@ -24,6 +24,7 @@ import org.snmp4j.security.SecurityLevel; import org.snmp4j.security.SecurityModel; import org.snmp4j.security.SecurityProtocols; import org.snmp4j.security.USM; +import org.snmp4j.smi.Address; import org.snmp4j.smi.GenericAddress; import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; @@ -36,6 +37,8 @@ import org.thingsboard.server.queue.util.TbSnmpTransportComponent; import org.thingsboard.server.transport.snmp.service.SnmpTransportService; import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; +import java.util.Optional; + @Service @TbSnmpTransportComponent @RequiredArgsConstructor @@ -97,7 +100,8 @@ public class SnmpAuthService { throw new UnsupportedOperationException("SNMP protocol version " + protocolVersion + " is not supported"); } - target.setAddress(GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getHost() + "/" + deviceTransportConfig.getPort())); + Address address = GenericAddress.parse(snmpUnderlyingProtocol + ":" + deviceTransportConfig.getHost() + "/" + deviceTransportConfig.getPort()); + target.setAddress(Optional.ofNullable(address).orElseThrow(() -> new IllegalArgumentException("Address of the SNMP device is invalid"))); target.setTimeout(profileTransportConfig.getTimeoutMs()); target.setRetries(profileTransportConfig.getRetries()); target.setVersion(protocolVersion.getCode()); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 94a18f3d54..f672e36190 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -16,11 +16,11 @@ package org.thingsboard.server.transport.snmp.service; import com.google.gson.JsonObject; +import lombok.Data; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.snmp4j.PDU; -import org.snmp4j.ScopedPDU; import org.snmp4j.Snmp; import org.snmp4j.TransportMapping; import org.snmp4j.event.ResponseEvent; @@ -28,26 +28,18 @@ import org.snmp4j.mp.MPv3; import org.snmp4j.security.SecurityModels; import org.snmp4j.security.SecurityProtocols; import org.snmp4j.security.USM; -import org.snmp4j.smi.Integer32; -import org.snmp4j.smi.Null; -import org.snmp4j.smi.OID; import org.snmp4j.smi.OctetString; -import org.snmp4j.smi.Variable; -import org.snmp4j.smi.VariableBinding; import org.snmp4j.transport.DefaultTcpTransportMapping; import org.snmp4j.transport.DefaultUdpTransportMapping; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.TbTransportService; -import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; -import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; -import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; -import org.thingsboard.server.common.data.transport.snmp.configs.RepeatingQueryingSnmpCommunicationConfig; -import org.thingsboard.server.common.data.transport.snmp.configs.SnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.RepeatingQueryingSnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.impl.ToDeviceRpcResponseQueryingSnmpCommunicationConfig; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.JsonConverter; @@ -60,44 +52,37 @@ import javax.annotation.PreDestroy; import java.io.IOException; import java.util.Collections; import java.util.EnumMap; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; -import java.util.function.BiConsumer; import java.util.stream.Collectors; @TbSnmpTransportComponent @Service @Slf4j +@RequiredArgsConstructor public class SnmpTransportService implements TbTransportService { private final TransportService transportService; + private final PduMapper pduMapper; @Getter private Snmp snmp; private ScheduledExecutorService queryingExecutor; private ExecutorService responseProcessingExecutor; - private final Map> responseProcessors = new EnumMap<>(SnmpCommunicationSpec.class); + private final Map responseProcessors = new EnumMap<>(SnmpCommunicationSpec.class); @Value("${transport.snmp.response_processing.parallelism_level}") private Integer responseProcessingParallelismLevel; @Value("${transport.snmp.underlying_protocol}") private String snmpUnderlyingProtocol; - public SnmpTransportService(TransportService transportService) { - this.transportService = transportService; - } - @PostConstruct private void init() throws IOException { - log.info("Initializing SNMP transport service"); - queryingExecutor = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors(), ThingsBoardThreadFactory.forName("snmp-querying")); responseProcessingExecutor = Executors.newWorkStealingPool(responseProcessingParallelismLevel); @@ -128,26 +113,23 @@ public class SnmpTransportService implements TbTransportService { public void createQueryingTasks(DeviceSessionContext sessionContext) { List> queryingTasks = sessionContext.getProfileTransportConfiguration().getCommunicationConfigs().stream() - .filter(config -> config.getSpec().isRepeatingQuerying()) + .filter(communicationConfig -> communicationConfig instanceof RepeatingQueryingSnmpCommunicationConfig) .map(config -> { RepeatingQueryingSnmpCommunicationConfig repeatingCommunicationConfig = (RepeatingQueryingSnmpCommunicationConfig) config; - return createQueryingTaskForConfig(sessionContext, repeatingCommunicationConfig); + Long queryingFrequency = repeatingCommunicationConfig.getQueryingFrequencyMs(); + + return queryingExecutor.scheduleWithFixedDelay(() -> { + try { + if (sessionContext.isActive()) { + sendRequest(sessionContext, repeatingCommunicationConfig); + } + } catch (Exception e) { + log.error("Failed to send SNMP request for device {}: {}", sessionContext.getDeviceId(), e.toString()); + } + }, queryingFrequency, queryingFrequency, TimeUnit.MILLISECONDS); }) .collect(Collectors.toList()); - sessionContext.setQueryingTasks(queryingTasks); - } - - private ScheduledFuture createQueryingTaskForConfig(DeviceSessionContext sessionContext, RepeatingQueryingSnmpCommunicationConfig communicationConfig) { - Long queryingFrequency = communicationConfig.getQueryingFrequencyMs(); - return queryingExecutor.scheduleWithFixedDelay(() -> { - try { - if (sessionContext.isActive()) { - sendRequest(sessionContext, communicationConfig); - } - } catch (Exception e) { - log.error("Failed to send SNMP request for device {}: {}", sessionContext.getDeviceId(), e.toString()); - } - }, queryingFrequency, queryingFrequency, TimeUnit.MILLISECONDS); + sessionContext.getQueryingTasks().addAll(queryingTasks); } public void cancelQueryingTasks(DeviceSessionContext sessionContext) { @@ -155,67 +137,23 @@ public class SnmpTransportService implements TbTransportService { sessionContext.getQueryingTasks().clear(); } - public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) throws IOException { + + public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) { sendRequest(sessionContext, communicationConfig, Collections.emptyMap()); } - public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) throws IOException { - PDU request = createPdu(sessionContext, communicationConfig, values); - executeRequest(sessionContext, request); - } + public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) { + PDU request = pduMapper.createPdu(sessionContext, communicationConfig, values); - private void executeRequest(DeviceSessionContext sessionContext, PDU request) throws IOException { if (request.size() > 0) { log.trace("Executing SNMP request for device {}. Variables bindings: {}", sessionContext.getDeviceId(), request.getVariableBindings()); - snmp.send(request, sessionContext.getTarget(), sessionContext.getDeviceProfile().getId(), sessionContext); - } - } - - private PDU createPdu(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) { - PDU pdu; - SnmpDeviceTransportConfiguration deviceTransportConfiguration = sessionContext.getDeviceTransportConfiguration(); - SnmpProtocolVersion snmpVersion = deviceTransportConfiguration.getProtocolVersion(); - switch (snmpVersion) { - case V1: - case V2C: - pdu = new PDU(); - break; - case V3: - ScopedPDU scopedPdu = new ScopedPDU(); - scopedPdu.setContextName(new OctetString(deviceTransportConfiguration.getContextName())); - scopedPdu.setContextEngineID(new OctetString(deviceTransportConfiguration.getEngineId())); - pdu = scopedPdu; - break; - default: - throw new UnsupportedOperationException("SNMP version " + snmpVersion + " is not supported"); + RequestInfo requestInfo = new RequestInfo(sessionContext.getDeviceProfile().getId(), communicationConfig); + try { + snmp.send(request, sessionContext.getTarget(), requestInfo, sessionContext); + } catch (IOException e) { + log.error("Failed to send SNMP request to device {}: {}", sessionContext.getDeviceId(), e.toString()); + } } - - pdu.setType(communicationConfig.getMethod().getCode()); - pdu.addAll(communicationConfig.getMappings().stream() - .filter(mapping -> values.isEmpty() || values.containsKey(mapping.getKey())) - .map(mapping -> Optional.ofNullable(values.get(mapping.getKey())) - .map(value -> { - Variable variable; - switch (mapping.getDataType()) { - case LONG: - try { - variable = new Integer32(Integer.parseInt(value)); - break; - } catch (NumberFormatException ignored) { - } - case DOUBLE: - case BOOLEAN: - case STRING: - case JSON: - default: - variable = new OctetString(value); - } - return new VariableBinding(new OID(mapping.getOid()), variable); - }) - .orElseGet(() -> new VariableBinding(new OID(mapping.getOid())))) - .collect(Collectors.toList())); - - return pdu; } @@ -223,7 +161,7 @@ public class SnmpTransportService implements TbTransportService { ((Snmp) event.getSource()).cancel(event.getRequest(), sessionContext); if (event.getError() != null) { - log.warn("Response error: {}", event.getError().getMessage(), event.getError()); + log.warn("SNMP response error: {}", event.getError().toString()); return; } @@ -232,108 +170,65 @@ public class SnmpTransportService implements TbTransportService { log.debug("No response from SNMP device {}, requestId: {}", sessionContext.getDeviceId(), event.getRequest().getRequestID()); return; } - DeviceProfileId deviceProfileId = (DeviceProfileId) event.getUserObject(); - log.debug("[{}] Processing SNMP response for device {} with device profile {}: {}", - response.getRequestID(), sessionContext.getDeviceId(), deviceProfileId, response); - responseProcessingExecutor.execute(() -> processResponse(sessionContext, response)); + RequestInfo requestInfo = (RequestInfo) event.getUserObject(); + responseProcessingExecutor.execute(() -> { + processResponse(sessionContext, response, requestInfo); + }); } - private void processResponse(DeviceSessionContext sessionContext, PDU responsePdu) { - Map mappings = new HashMap<>(); - Map configs = new HashMap<>(); - Map responses = new EnumMap<>(SnmpCommunicationSpec.class); - - for (SnmpCommunicationConfig config : sessionContext.getProfileTransportConfiguration().getCommunicationConfigs()) { - for (SnmpMapping mapping : config.getMappings()) { - OID oid = new OID(mapping.getOid()); - mappings.put(oid, mapping); - configs.put(oid, config); - } - responses.put(config.getSpec(), new JsonObject()); - } - - for (int i = 0; i < responsePdu.size(); i++) { - VariableBinding variableBinding = responsePdu.get(i); - log.trace("Processing variable binding {}: {}", i, variableBinding); - - if (variableBinding.getVariable() instanceof Null) { - log.debug("Response variable is empty"); - continue; - } - - OID oid = variableBinding.getOid(); - if (!mappings.containsKey(oid)) { - log.debug("No SNMP mapping for oid {}", oid); - continue; - } - - SnmpCommunicationSpec spec = configs.get(oid).getSpec(); - if (!responseProcessors.containsKey(spec)) { - log.debug("No response processor found for spec {}", spec); - continue; - } - - SnmpMapping mapping = mappings.get(oid); - processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), responses.get(spec)); + private void processResponse(DeviceSessionContext sessionContext, PDU response, RequestInfo requestInfo) { + ResponseProcessor responseProcessor = responseProcessors.get(requestInfo.getCommunicationConfig().getSpec()); + if (responseProcessor == null) { + return; } - if (responses.values().stream().allMatch(response -> response.entrySet().isEmpty())) { - log.debug("No values is the SNMP response for device {}. Request id: {}", sessionContext.getDeviceId(), responsePdu.getRequestID()); + JsonObject responseData = pduMapper.processPdu(response, sessionContext, requestInfo.getCommunicationConfig()); + if (responseData.entrySet().isEmpty()) { + log.debug("No values is the SNMP response for device {}. Request id: {}", sessionContext.getDeviceId(), response.getRequestID()); return; } - responses.forEach((spec, response) -> { - Optional.ofNullable(responseProcessors.get(spec)) - .ifPresent(responseProcessor -> { - if (!response.entrySet().isEmpty()) { - responseProcessor.accept(response, sessionContext); - } - }); - }); - + responseProcessor.process(responseData, sessionContext); reportActivity(sessionContext.getSessionInfo()); } private void configureResponseProcessors() { responseProcessors.put(SnmpCommunicationSpec.TELEMETRY_QUERYING, (response, sessionContext) -> { TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(response); - transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, TransportServiceCallback.EMPTY); - log.debug("Posted telemetry for device {}: {}", sessionContext.getDeviceId(), response); + transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, null); + log.debug("Posted telemetry for SNMP device {}: {}", sessionContext.getDeviceId(), response); }); responseProcessors.put(SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING, (response, sessionContext) -> { TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(response); - transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, TransportServiceCallback.EMPTY); - log.debug("Posted attributes for device {}: {}", sessionContext.getDeviceId(), response); + transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, null); + log.debug("Posted attributes for SNMP device {}: {}", sessionContext.getDeviceId(), response); + }); + + responseProcessors.put(SnmpCommunicationSpec.TO_DEVICE_RPC_RESPONSE_QUERYING, (response, sessionContext) -> { + String rpcResponse = response.get(ToDeviceRpcResponseQueryingSnmpCommunicationConfig.RPC_RESPONSE_KEY_NAME).getAsString(); + TransportProtos.ToDeviceRpcResponseMsg rpcResponseMsg = TransportProtos.ToDeviceRpcResponseMsg.newBuilder() + .setPayload(rpcResponse) + .build(); + transportService.process(sessionContext.getSessionInfo(), rpcResponseMsg, null); + log.debug("Processed RPC response from device {}: {}", sessionContext.getDeviceId(), rpcResponse); }); + +// responseProcessors.put(, (response, sessionContext) -> { +// TransportProtos.ClaimDeviceMsg claimDeviceMsg = JsonConverter.convertToClaimDeviceProto(sessionContext.getDeviceId(), response); +// transportService.process(sessionContext.getSessionInfo(), claimDeviceMsg, null); +// }); } private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { transportService.process(sessionInfo, TransportProtos.SubscriptionInfoProto.newBuilder() - .setAttributeSubscription(false) - .setRpcSubscription(false) + .setAttributeSubscription(true) + .setRpcSubscription(true) .setLastActivityTime(System.currentTimeMillis()) .build(), TransportServiceCallback.EMPTY); } - private void processValue(String key, DataType dataType, String value, JsonObject result) { - if (StringUtils.isEmpty(value)) return; - - switch (dataType) { - case LONG: - result.addProperty(key, Long.parseLong(value)); - break; - case BOOLEAN: - result.addProperty(key, Boolean.parseBoolean(value)); - break; - case DOUBLE: - result.addProperty(key, Double.parseDouble(value)); - break; - default: - result.addProperty(key, value); - } - } @Override public String getName() { @@ -358,4 +253,15 @@ public class SnmpTransportService implements TbTransportService { } log.info("SNMP transport stopped!"); } + + @Data + private static class RequestInfo { + private final DeviceProfileId deviceProfileId; + private final SnmpCommunicationConfig communicationConfig; + } + + private interface ResponseProcessor { + void process(JsonObject responseData, DeviceSessionContext sessionContext); + } + } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index b3a38545a8..82fdc037ad 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -27,6 +27,8 @@ import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfigu import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.impl.ToDeviceRpcCommandSettingSnmpCommunicationConfig; import org.thingsboard.server.common.transport.SessionMsgListener; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; @@ -36,13 +38,13 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponse import org.thingsboard.server.gen.transport.TransportProtos.SessionCloseNotificationProto; import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.transport.snmp.SnmpAuthService; import org.thingsboard.server.transport.snmp.SnmpTransportContext; -import org.thingsboard.server.transport.snmp.service.SnmpTransportService; +import java.io.IOException; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicInteger; @@ -64,16 +66,12 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S private final SnmpTransportContext snmpTransportContext; - @Getter - @Setter - private long previousRequestExecutedAt = 0; private final AtomicInteger msgIdSeq = new AtomicInteger(0); @Getter private boolean isActive = true; @Getter - @Setter - private List> queryingTasks = new LinkedList<>(); + private final List> queryingTasks = new LinkedList<>(); public DeviceSessionContext(Device device, DeviceProfile deviceProfile, String token, SnmpDeviceProfileTransportConfiguration profileTransportConfiguration, @@ -116,7 +114,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S public void initializeTarget(SnmpDeviceProfileTransportConfiguration profileTransportConfig, SnmpDeviceTransportConfiguration deviceTransportConfig) throws Exception { log.trace("Initializing target for SNMP session of device {}", device); this.target = snmpTransportContext.getSnmpAuthService().setUpSnmpTarget(profileTransportConfig, deviceTransportConfig); - log.info("SNMP target initialized: {}", target); + log.debug("SNMP target initialized: {}", target); } public void close() { @@ -138,20 +136,14 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { - profileTransportConfiguration.getCommunicationConfigs().stream() - .filter(config -> config.getSpec() == SnmpCommunicationSpec.SHARED_ATTRIBUTES_SETTING) - .findFirst() + getCommunicationConfigForSpec(SnmpCommunicationSpec.SHARED_ATTRIBUTES_SETTING) .ifPresent(communicationConfig -> { Map sharedAttributes = JsonConverter.toJson(attributeUpdateNotification).entrySet().stream() .collect(Collectors.toMap( Map.Entry::getKey, entry -> entry.getValue().isJsonPrimitive() ? entry.getValue().getAsString() : entry.getValue().toString() )); - try { - snmpTransportContext.getSnmpTransportService().sendRequest(this, communicationConfig, sharedAttributes); - } catch (Exception e) { - log.error("Failed to send request with shared attributes to SNMP device {}: {}", getDeviceId(), e.getMessage()); - } + snmpTransportContext.getSnmpTransportService().sendRequest(this, communicationConfig, sharedAttributes); }); } @@ -161,9 +153,23 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) { + getCommunicationConfigForSpec(SnmpCommunicationSpec.TO_DEVICE_RPC_COMMAND_SETTING) + .ifPresent(communicationConfig -> { + String value = JsonConverter.toJson(toDeviceRequest, true).toString(); + snmpTransportContext.getSnmpTransportService().sendRequest( + this, communicationConfig, + Map.of(ToDeviceRpcCommandSettingSnmpCommunicationConfig.RPC_COMMAND_KEY_NAME, value) + ); + }); } @Override public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) { } + + private Optional getCommunicationConfigForSpec(SnmpCommunicationSpec spec) { + return profileTransportConfiguration.getCommunicationConfigs().stream() + .filter(config -> config.getSpec() == spec) + .findFirst(); + } } From a1e4bbbb31cca6360f77dd838d83872e646ed917 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 20 Apr 2021 17:58:15 +0300 Subject: [PATCH 13/32] Implement RPC for SNMP; refactor --- .../SnmpDeviceTransportConfiguration.java | 2 +- ...mpDeviceProfileTransportConfiguration.java | 2 +- .../transport/snmp/SnmpCommunicationSpec.java | 3 +- ...CommandSettingSnmpCommunicationConfig.java | 59 -------- ...sponseQueryingSnmpCommunicationConfig.java | 60 -------- .../transport/snmp/SnmpTransportContext.java | 1 + .../{PduMapper.java => PduService.java} | 91 +++++++----- .../snmp/service/SnmpTransportService.java | 135 ++++++++++++++---- .../snmp/session/DeviceSessionContext.java | 33 +---- .../transport/adaptor/JsonConverter.java | 11 +- 10 files changed, 175 insertions(+), 222 deletions(-) delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java delete mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java rename common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/{PduMapper.java => PduService.java} (76%) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java index 68a9d1f218..a7bc143d81 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/data/SnmpDeviceTransportConfiguration.java @@ -40,7 +40,7 @@ public class SnmpDeviceTransportConfiguration implements DeviceTransportConfigur private String community; /* - * For SNMP v3 with User Based Security Model + * For SNMP v3 * */ private String username; private String securityName; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java index bbf074eeda..0b8efaf8c7 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/SnmpDeviceProfileTransportConfiguration.java @@ -44,7 +44,7 @@ public class SnmpDeviceProfileTransportConfiguration implements DeviceProfileTra @JsonIgnore private boolean isValid() { return timeoutMs != null && timeoutMs >= 0 && retries != null && retries >= 0 - && communicationConfigs != null && !communicationConfigs.isEmpty() + && communicationConfigs != null && communicationConfigs.stream().allMatch(config -> config != null && config.isValid()) && communicationConfigs.stream().flatMap(config -> config.getAllMappings().stream()).map(SnmpMapping::getOid) .distinct().count() == communicationConfigs.stream().mapToInt(config -> config.getAllMappings().size()).sum(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java index a6643ecf1e..8d87144ae8 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/SnmpCommunicationSpec.java @@ -21,6 +21,5 @@ public enum SnmpCommunicationSpec { CLIENT_ATTRIBUTES_QUERYING, SHARED_ATTRIBUTES_SETTING, - TO_DEVICE_RPC_COMMAND_SETTING, - TO_DEVICE_RPC_RESPONSE_QUERYING + TO_DEVICE_RPC_REQUEST, } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java deleted file mode 100644 index 97f3de047b..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcCommandSettingSnmpCommunicationConfig.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.transport.snmp.config.impl; - -import lombok.Data; -import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; -import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; -import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; -import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -@Data -public class ToDeviceRpcCommandSettingSnmpCommunicationConfig implements SnmpCommunicationConfig { - private SnmpMapping mapping; - - @Override - public SnmpCommunicationSpec getSpec() { - return SnmpCommunicationSpec.TO_DEVICE_RPC_COMMAND_SETTING; - } - - @Override - public SnmpMethod getMethod() { - return SnmpMethod.SET; - } - - public void setMapping(SnmpMapping mapping) { - this.mapping = mapping != null ? new SnmpMapping(mapping.getOid(), RPC_COMMAND_KEY_NAME, DataType.STRING) : null; - } - - @Override - public List getAllMappings() { - return Collections.singletonList(mapping); - } - - @Override - public boolean isValid() { - return mapping != null && mapping.isValid(); - } - - public static final String RPC_COMMAND_KEY_NAME = "rpcCommand"; - -} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java deleted file mode 100644 index 18bce94980..0000000000 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcResponseQueryingSnmpCommunicationConfig.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Copyright © 2016-2021 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.data.transport.snmp.config.impl; - -import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; -import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; -import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; -import org.thingsboard.server.common.data.transport.snmp.config.RepeatingQueryingSnmpCommunicationConfig; - -import java.util.Collections; -import java.util.List; - -@EqualsAndHashCode(callSuper = true) -@Data -public class ToDeviceRpcResponseQueryingSnmpCommunicationConfig extends RepeatingQueryingSnmpCommunicationConfig { - private SnmpMapping mapping; - - @Override - public SnmpCommunicationSpec getSpec() { - return SnmpCommunicationSpec.TO_DEVICE_RPC_RESPONSE_QUERYING; - } - - @Override - public SnmpMethod getMethod() { - return SnmpMethod.GET; - } - - public void setMapping(SnmpMapping mapping) { - this.mapping = mapping != null ? new SnmpMapping(mapping.getOid(), RPC_RESPONSE_KEY_NAME, DataType.STRING) : null; - } - - @Override - public List getAllMappings() { - return Collections.singletonList(mapping); - } - - @Override - public boolean isValid() { - return true; - } - - public static final String RPC_RESPONSE_KEY_NAME = "rpcResponse"; - -} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index d66d09ab0b..68d4dd3933 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -152,6 +152,7 @@ public class SnmpTransportContext extends TransportContext { } } catch (Exception e) { log.error("Failed to update session for SNMP device {}: {}", sessionContext.getDeviceId(), e.getMessage()); + destroyDeviceSession(sessionContext); } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduMapper.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduService.java similarity index 76% rename from common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduMapper.java rename to common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduService.java index a3b5f77dfb..720b1e7ae3 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduMapper.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduService.java @@ -29,6 +29,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; import org.thingsboard.server.common.data.transport.snmp.SnmpProtocolVersion; import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; @@ -45,32 +46,16 @@ import java.util.stream.IntStream; @TbSnmpTransportComponent @Service @Slf4j -public class PduMapper { +public class PduService { public PDU createPdu(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) { - PDU pdu; - SnmpDeviceTransportConfiguration deviceTransportConfiguration = sessionContext.getDeviceTransportConfiguration(); - SnmpProtocolVersion snmpVersion = deviceTransportConfiguration.getProtocolVersion(); - switch (snmpVersion) { - case V1: - case V2C: - pdu = new PDU(); - break; - case V3: - ScopedPDU scopedPdu = new ScopedPDU(); - scopedPdu.setContextName(new OctetString(deviceTransportConfiguration.getContextName())); - scopedPdu.setContextEngineID(new OctetString(deviceTransportConfiguration.getEngineId())); - pdu = scopedPdu; - break; - default: - throw new UnsupportedOperationException("SNMP version " + snmpVersion + " is not supported"); - } + PDU pdu = setUpPdu(sessionContext); pdu.setType(communicationConfig.getMethod().getCode()); pdu.addAll(communicationConfig.getAllMappings().stream() .filter(mapping -> values.isEmpty() || values.containsKey(mapping.getKey())) .map(mapping -> Optional.ofNullable(values.get(mapping.getKey())) .map(value -> { - Variable variable = toSnmpVariable(mapping, value); + Variable variable = toSnmpVariable(value, mapping.getDataType()); return new VariableBinding(new OID(mapping.getOid()), variable); }) .orElseGet(() -> new VariableBinding(new OID(mapping.getOid())))) @@ -79,9 +64,20 @@ public class PduMapper { return pdu; } - private Variable toSnmpVariable(SnmpMapping mapping, String value) { + public PDU createSingleVariablePdu(DeviceSessionContext sessionContext, SnmpMethod snmpMethod, String oid, String value, DataType dataType) { + PDU pdu = setUpPdu(sessionContext); + pdu.setType(snmpMethod.getCode()); + + Variable variable = value == null ? Null.instance : toSnmpVariable(value, dataType); + pdu.add(new VariableBinding(new OID(oid), variable)); + + return pdu; + } + + private Variable toSnmpVariable(String value, DataType dataType) { + dataType = dataType == null ? DataType.STRING : dataType; Variable variable; - switch (mapping.getDataType()) { + switch (dataType) { case LONG: try { variable = new Integer32(Integer.parseInt(value)); @@ -98,37 +94,62 @@ public class PduMapper { return variable; } + private PDU setUpPdu(DeviceSessionContext sessionContext) { + PDU pdu; + SnmpDeviceTransportConfiguration deviceTransportConfiguration = sessionContext.getDeviceTransportConfiguration(); + SnmpProtocolVersion snmpVersion = deviceTransportConfiguration.getProtocolVersion(); + switch (snmpVersion) { + case V1: + case V2C: + pdu = new PDU(); + break; + case V3: + ScopedPDU scopedPdu = new ScopedPDU(); + scopedPdu.setContextName(new OctetString(deviceTransportConfiguration.getContextName())); + scopedPdu.setContextEngineID(new OctetString(deviceTransportConfiguration.getEngineId())); + pdu = scopedPdu; + break; + default: + throw new UnsupportedOperationException("SNMP version " + snmpVersion + " is not supported"); + } + return pdu; + } - public JsonObject processPdu(PDU pdu, DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) { - List variablesBindings = IntStream.range(0, pdu.size()) - .mapToObj(pdu::get) - .filter(Objects::nonNull) - .filter(variableBinding -> !(variableBinding.getVariable() instanceof Null)) - .collect(Collectors.toList()); - JsonObject data = new JsonObject(); + + public JsonObject processPdu(PDU pdu, List responseMappings) { + Map values = processPdu(pdu); Map mappings = new HashMap<>(); - for (SnmpMapping mapping : communicationConfig.getAllMappings()) { - OID oid = new OID(mapping.getOid()); - mappings.put(oid, mapping); + if (responseMappings != null) { + for (SnmpMapping mapping : responseMappings) { + OID oid = new OID(mapping.getOid()); + mappings.put(oid, mapping); + } } - variablesBindings.forEach(variableBinding -> { - log.trace("Processing variable binding: {}", variableBinding); + JsonObject data = new JsonObject(); + values.forEach((oid, value) -> { + log.trace("Processing variable binding: {} - {}", oid, value); - OID oid = variableBinding.getOid(); SnmpMapping mapping = mappings.get(oid); if (mapping == null) { log.debug("No SNMP mapping for oid {}", oid); return; } - processValue(mapping.getKey(), mapping.getDataType(), variableBinding.toValueString(), data); + processValue(mapping.getKey(), mapping.getDataType(), value, data); }); return data; } + public Map processPdu(PDU pdu) { + return IntStream.range(0, pdu.size()) + .mapToObj(pdu::get) + .filter(Objects::nonNull) + .filter(variableBinding -> !(variableBinding.getVariable() instanceof Null)) + .collect(Collectors.toMap(VariableBinding::getOid, VariableBinding::toValueString)); + } private void processValue(String key, DataType dataType, String value, JsonObject result) { switch (dataType) { diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index f672e36190..5bfee14c41 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.transport.snmp.service; +import com.google.gson.JsonElement; import com.google.gson.JsonObject; import lombok.Data; import lombok.Getter; @@ -35,11 +36,12 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.TbTransportService; -import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.kv.DataType; import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.SnmpMapping; +import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; import org.thingsboard.server.common.data.transport.snmp.config.RepeatingQueryingSnmpCommunicationConfig; import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; -import org.thingsboard.server.common.data.transport.snmp.config.impl.ToDeviceRpcResponseQueryingSnmpCommunicationConfig; import org.thingsboard.server.common.transport.TransportService; import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.JsonConverter; @@ -50,10 +52,12 @@ import org.thingsboard.server.transport.snmp.session.DeviceSessionContext; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -67,13 +71,14 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class SnmpTransportService implements TbTransportService { private final TransportService transportService; - private final PduMapper pduMapper; + private final PduService pduService; @Getter private Snmp snmp; private ScheduledExecutorService queryingExecutor; private ExecutorService responseProcessingExecutor; + private final Map responseDataMappers = new EnumMap<>(SnmpCommunicationSpec.class); private final Map responseProcessors = new EnumMap<>(SnmpCommunicationSpec.class); @Value("${transport.snmp.response_processing.parallelism_level}") @@ -87,6 +92,7 @@ public class SnmpTransportService implements TbTransportService { responseProcessingExecutor = Executors.newWorkStealingPool(responseProcessingParallelismLevel); initializeSnmp(); + configureResponseDataMappers(); configureResponseProcessors(); log.info("SNMP transport service initialized"); @@ -138,16 +144,19 @@ public class SnmpTransportService implements TbTransportService { } - public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) { + private void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig) { sendRequest(sessionContext, communicationConfig, Collections.emptyMap()); } - public void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) { - PDU request = pduMapper.createPdu(sessionContext, communicationConfig, values); + private void sendRequest(DeviceSessionContext sessionContext, SnmpCommunicationConfig communicationConfig, Map values) { + PDU request = pduService.createPdu(sessionContext, communicationConfig, values); + RequestInfo requestInfo = new RequestInfo(communicationConfig.getSpec(), communicationConfig.getAllMappings()); + sendRequest(sessionContext, request, requestInfo); + } + private void sendRequest(DeviceSessionContext sessionContext, PDU request, RequestInfo requestInfo) { if (request.size() > 0) { log.trace("Executing SNMP request for device {}. Variables bindings: {}", sessionContext.getDeviceId(), request.getVariableBindings()); - RequestInfo requestInfo = new RequestInfo(sessionContext.getDeviceProfile().getId(), communicationConfig); try { snmp.send(request, sessionContext.getTarget(), requestInfo, sessionContext); } catch (IOException e) { @@ -156,6 +165,39 @@ public class SnmpTransportService implements TbTransportService { } } + public void onAttributeUpdate(DeviceSessionContext sessionContext, TransportProtos.AttributeUpdateNotificationMsg attributeUpdateNotification) { + sessionContext.getProfileTransportConfiguration().getCommunicationConfigs().stream() + .filter(config -> config.getSpec() == SnmpCommunicationSpec.SHARED_ATTRIBUTES_SETTING) + .findFirst() + .ifPresent(communicationConfig -> { + Map sharedAttributes = JsonConverter.toJson(attributeUpdateNotification).entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> entry.getValue().isJsonPrimitive() ? entry.getValue().getAsString() : entry.getValue().toString() + )); + sendRequest(sessionContext, communicationConfig, sharedAttributes); + }); + } + + public void onToDeviceRpcRequest(DeviceSessionContext sessionContext, TransportProtos.ToDeviceRpcRequestMsg toDeviceRpcRequestMsg) { + SnmpMethod snmpMethod = SnmpMethod.valueOf(toDeviceRpcRequestMsg.getMethodName()); + JsonObject params = JsonConverter.parse(toDeviceRpcRequestMsg.getParams()).getAsJsonObject(); + + String oid = Optional.ofNullable(params.get("oid")).map(JsonElement::getAsString).orElse(null); + String value = Optional.ofNullable(params.get("value")).map(JsonElement::getAsString).orElse(null); + DataType dataType = Optional.ofNullable(params.get("dataType")).map(e -> DataType.valueOf(e.getAsString())).orElse(DataType.STRING); + + if (oid == null || oid.isEmpty()) { + throw new IllegalArgumentException("OID in to-device RPC request is not specified"); + } + if (value == null && snmpMethod == SnmpMethod.SET) { + throw new IllegalArgumentException("Value must be specified for SNMP method 'SET'"); + } + + PDU request = pduService.createSingleVariablePdu(sessionContext, snmpMethod, oid, value, dataType); + sendRequest(sessionContext, request, new RequestInfo(toDeviceRpcRequestMsg.getRequestId(), SnmpCommunicationSpec.TO_DEVICE_RPC_REQUEST)); + } + public void processResponseEvent(DeviceSessionContext sessionContext, ResponseEvent event) { ((Snmp) event.getSource()).cancel(event.getRequest(), sessionContext); @@ -178,47 +220,59 @@ public class SnmpTransportService implements TbTransportService { } private void processResponse(DeviceSessionContext sessionContext, PDU response, RequestInfo requestInfo) { - ResponseProcessor responseProcessor = responseProcessors.get(requestInfo.getCommunicationConfig().getSpec()); - if (responseProcessor == null) { - return; - } + ResponseProcessor responseProcessor = responseProcessors.get(requestInfo.getCommunicationSpec()); + if (responseProcessor == null) return; + + JsonObject responseData = responseDataMappers.get(requestInfo.getCommunicationSpec()).map(response, requestInfo); - JsonObject responseData = pduMapper.processPdu(response, sessionContext, requestInfo.getCommunicationConfig()); if (responseData.entrySet().isEmpty()) { log.debug("No values is the SNMP response for device {}. Request id: {}", sessionContext.getDeviceId(), response.getRequestID()); return; } - responseProcessor.process(responseData, sessionContext); + responseProcessor.process(responseData, requestInfo, sessionContext); reportActivity(sessionContext.getSessionInfo()); } + private void configureResponseDataMappers() { + responseDataMappers.put(SnmpCommunicationSpec.TO_DEVICE_RPC_REQUEST, (pdu, requestInfo) -> { + JsonObject responseData = new JsonObject(); + pduService.processPdu(pdu).forEach((oid, value) -> { + responseData.addProperty(oid.toDottedString(), value); + }); + return responseData; + }); + + ResponseDataMapper defaultResponseDataMapper = (pdu, requestInfo) -> { + return pduService.processPdu(pdu, requestInfo.getResponseMappings()); + }; + Arrays.stream(SnmpCommunicationSpec.values()) + .forEach(communicationSpec -> { + responseDataMappers.putIfAbsent(communicationSpec, defaultResponseDataMapper); + }); + } + private void configureResponseProcessors() { - responseProcessors.put(SnmpCommunicationSpec.TELEMETRY_QUERYING, (response, sessionContext) -> { - TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(response); + responseProcessors.put(SnmpCommunicationSpec.TELEMETRY_QUERYING, (responseData, requestInfo, sessionContext) -> { + TransportProtos.PostTelemetryMsg postTelemetryMsg = JsonConverter.convertToTelemetryProto(responseData); transportService.process(sessionContext.getSessionInfo(), postTelemetryMsg, null); - log.debug("Posted telemetry for SNMP device {}: {}", sessionContext.getDeviceId(), response); + log.debug("Posted telemetry for SNMP device {}: {}", sessionContext.getDeviceId(), responseData); }); - responseProcessors.put(SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING, (response, sessionContext) -> { - TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(response); + responseProcessors.put(SnmpCommunicationSpec.CLIENT_ATTRIBUTES_QUERYING, (responseData, requestInfo, sessionContext) -> { + TransportProtos.PostAttributeMsg postAttributesMsg = JsonConverter.convertToAttributesProto(responseData); transportService.process(sessionContext.getSessionInfo(), postAttributesMsg, null); - log.debug("Posted attributes for SNMP device {}: {}", sessionContext.getDeviceId(), response); + log.debug("Posted attributes for SNMP device {}: {}", sessionContext.getDeviceId(), responseData); }); - responseProcessors.put(SnmpCommunicationSpec.TO_DEVICE_RPC_RESPONSE_QUERYING, (response, sessionContext) -> { - String rpcResponse = response.get(ToDeviceRpcResponseQueryingSnmpCommunicationConfig.RPC_RESPONSE_KEY_NAME).getAsString(); + responseProcessors.put(SnmpCommunicationSpec.TO_DEVICE_RPC_REQUEST, (responseData, requestInfo, sessionContext) -> { TransportProtos.ToDeviceRpcResponseMsg rpcResponseMsg = TransportProtos.ToDeviceRpcResponseMsg.newBuilder() - .setPayload(rpcResponse) + .setRequestId(requestInfo.getRequestId()) + .setPayload(JsonConverter.toJson(responseData)) .build(); transportService.process(sessionContext.getSessionInfo(), rpcResponseMsg, null); - log.debug("Processed RPC response from device {}: {}", sessionContext.getDeviceId(), rpcResponse); + log.debug("Posted RPC response {} for device {}", responseData, sessionContext.getDeviceId()); }); - -// responseProcessors.put(, (response, sessionContext) -> { -// TransportProtos.ClaimDeviceMsg claimDeviceMsg = JsonConverter.convertToClaimDeviceProto(sessionContext.getDeviceId(), response); -// transportService.process(sessionContext.getSessionInfo(), claimDeviceMsg, null); -// }); } private void reportActivity(TransportProtos.SessionInfoProto sessionInfo) { @@ -256,12 +310,31 @@ public class SnmpTransportService implements TbTransportService { @Data private static class RequestInfo { - private final DeviceProfileId deviceProfileId; - private final SnmpCommunicationConfig communicationConfig; + private Integer requestId; + private SnmpCommunicationSpec communicationSpec; + private List responseMappings; + + public RequestInfo(Integer requestId, SnmpCommunicationSpec communicationSpec) { + this.requestId = requestId; + this.communicationSpec = communicationSpec; + } + + public RequestInfo(SnmpCommunicationSpec communicationSpec) { + this.communicationSpec = communicationSpec; + } + + public RequestInfo(SnmpCommunicationSpec communicationSpec, List responseMappings) { + this.communicationSpec = communicationSpec; + this.responseMappings = responseMappings; + } + } + + private interface ResponseDataMapper { + JsonObject map(PDU pdu, RequestInfo requestInfo); } private interface ResponseProcessor { - void process(JsonObject responseData, DeviceSessionContext sessionContext); + void process(JsonObject responseData, RequestInfo requestInfo, DeviceSessionContext sessionContext); } } diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java index 82fdc037ad..a59ce020ba 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/session/DeviceSessionContext.java @@ -26,11 +26,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.device.data.SnmpDeviceTransportConfiguration; import org.thingsboard.server.common.data.device.profile.SnmpDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; -import org.thingsboard.server.common.data.transport.snmp.config.SnmpCommunicationConfig; -import org.thingsboard.server.common.data.transport.snmp.config.impl.ToDeviceRpcCommandSettingSnmpCommunicationConfig; import org.thingsboard.server.common.transport.SessionMsgListener; -import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.common.transport.session.DeviceAwareSessionContext; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; @@ -40,15 +36,11 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; import org.thingsboard.server.transport.snmp.SnmpTransportContext; -import java.io.IOException; import java.util.LinkedList; import java.util.List; -import java.util.Map; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; @Slf4j public class DeviceSessionContext extends DeviceAwareSessionContext implements SessionMsgListener, ResponseListener { @@ -136,15 +128,7 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onAttributeUpdate(AttributeUpdateNotificationMsg attributeUpdateNotification) { - getCommunicationConfigForSpec(SnmpCommunicationSpec.SHARED_ATTRIBUTES_SETTING) - .ifPresent(communicationConfig -> { - Map sharedAttributes = JsonConverter.toJson(attributeUpdateNotification).entrySet().stream() - .collect(Collectors.toMap( - Map.Entry::getKey, - entry -> entry.getValue().isJsonPrimitive() ? entry.getValue().getAsString() : entry.getValue().toString() - )); - snmpTransportContext.getSnmpTransportService().sendRequest(this, communicationConfig, sharedAttributes); - }); + snmpTransportContext.getSnmpTransportService().onAttributeUpdate(this, attributeUpdateNotification); } @Override @@ -153,23 +137,10 @@ public class DeviceSessionContext extends DeviceAwareSessionContext implements S @Override public void onToDeviceRpcRequest(ToDeviceRpcRequestMsg toDeviceRequest) { - getCommunicationConfigForSpec(SnmpCommunicationSpec.TO_DEVICE_RPC_COMMAND_SETTING) - .ifPresent(communicationConfig -> { - String value = JsonConverter.toJson(toDeviceRequest, true).toString(); - snmpTransportContext.getSnmpTransportService().sendRequest( - this, communicationConfig, - Map.of(ToDeviceRpcCommandSettingSnmpCommunicationConfig.RPC_COMMAND_KEY_NAME, value) - ); - }); + snmpTransportContext.getSnmpTransportService().onToDeviceRpcRequest(this, toDeviceRequest); } @Override public void onToServerRpcResponse(ToServerRpcResponseMsg toServerResponse) { } - - private Optional getCommunicationConfigForSpec(SnmpCommunicationSpec spec) { - return profileTransportConfiguration.getCommunicationConfigs().stream() - .filter(config -> config.getSpec() == spec) - .findFirst(); - } } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index fe2f3500a6..fce1c2c9b5 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -558,6 +558,14 @@ public class JsonConverter { } } + public static JsonElement parse(String json) { + return JSON_PARSER.parse(json); + } + + public static String toJson(JsonElement element) { + return GSON.toJson(element); + } + public static void setTypeCastEnabled(boolean enabled) { isTypeCastEnabled = enabled; } @@ -599,8 +607,7 @@ public class JsonConverter { .build(); } - private static TransportProtos.ProvisionDeviceCredentialsMsg buildProvisionDeviceCredentialsMsg(String - provisionKey, String provisionSecret) { + private static TransportProtos.ProvisionDeviceCredentialsMsg buildProvisionDeviceCredentialsMsg(String provisionKey, String provisionSecret) { return TransportProtos.ProvisionDeviceCredentialsMsg.newBuilder() .setProvisionDeviceKey(provisionKey) .setProvisionDeviceSecret(provisionSecret) From 42438e4c95525242799a8c05e9eeb218013481ae Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 26 Apr 2021 13:15:37 +0300 Subject: [PATCH 14/32] Configure Docker setup for SNMP transport --- docker/.env | 1 + docker/docker-compose.aws-sqs.yml | 5 + docker/docker-compose.confluent.yml | 3 + docker/docker-compose.kafka.yml | 5 + docker/docker-compose.postgres.volumes.yml | 6 + docker/docker-compose.pubsub.yml | 5 + docker/docker-compose.rabbitmq.yml | 5 + docker/docker-compose.service-bus.yml | 5 + docker/docker-compose.yml | 12 + docker/docker-create-log-folders.sh | 2 + docker/tb-snmp-transport.env | 2 + .../server/msa/ThingsBoardDbInstaller.java | 11 +- .../src/main/resources/tb-snmp-transport.yml | 214 ++---------------- 13 files changed, 84 insertions(+), 192 deletions(-) create mode 100644 docker/tb-snmp-transport.env diff --git a/docker/.env b/docker/.env index f7b30aafe0..e63e1dbf8d 100644 --- a/docker/.env +++ b/docker/.env @@ -9,6 +9,7 @@ MQTT_TRANSPORT_DOCKER_NAME=tb-mqtt-transport HTTP_TRANSPORT_DOCKER_NAME=tb-http-transport COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport +SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VERSION=latest diff --git a/docker/docker-compose.aws-sqs.yml b/docker/docker-compose.aws-sqs.yml index f70ad9410c..bbc5a3dcf6 100644 --- a/docker/docker-compose.aws-sqs.yml +++ b/docker/docker-compose.aws-sqs.yml @@ -74,3 +74,8 @@ services: - queue-aws-sqs.env depends_on: - zookeeper + tb-snmp-transport: + env_file: + - queue-aws-sqs.env + depends_on: + - zookeeper diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml index e048ed0524..b0d50d47d9 100644 --- a/docker/docker-compose.confluent.yml +++ b/docker/docker-compose.confluent.yml @@ -58,3 +58,6 @@ services: tb-lwm2m-transport: env_file: - queue-confluent.env + tb-snmp-transport: + env_file: + - queue-confluent.env diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index e46df318c5..481c97def2 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -85,3 +85,8 @@ services: - queue-kafka.env depends_on: - kafka + tb-snmp-transport: + env_file: + - queue-kafka.env + depends_on: + - kafka diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 2819bdf62d..510f5426d8 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -50,6 +50,9 @@ services: tb-mqtt-transport2: volumes: - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-snmp-transport: + volumes: + - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport volumes: postgres-db-volume: @@ -70,3 +73,6 @@ volumes: tb-mqtt-transport-log-volume: external: true name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} + tb-snmp-transport-log-volume: + external: true + name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index 3041768951..cb09cdb266 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -74,3 +74,8 @@ services: - queue-pubsub.env depends_on: - zookeeper + tb-snmp-transport: + env_file: + - queue-pubsub.env + depends_on: + - zookeeper diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index b4d4b6cf3e..d070a8c1db 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -74,3 +74,8 @@ services: - queue-rabbitmq.env depends_on: - zookeeper + tb-snmp-transport: + env_file: + - queue-rabbitmq.env + depends_on: + - zookeeper diff --git a/docker/docker-compose.service-bus.yml b/docker/docker-compose.service-bus.yml index 29004d61fa..e2cadb3edb 100644 --- a/docker/docker-compose.service-bus.yml +++ b/docker/docker-compose.service-bus.yml @@ -72,3 +72,8 @@ services: - queue-service-bus.env depends_on: - zookeeper + tb-snmp-transport: + env_file: + - queue-service-bus.env + depends_on: + - zookeeper diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d2440329ed..5da49759c2 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -217,6 +217,18 @@ services: - ./tb-transports/lwm2m/log:/var/log/tb-lwm2m-transport depends_on: - zookeeper + tb-snmp-transport: + restart: always + image: "${DOCKER_REPO}/${SNMP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" + environment: + TB_SERVICE_ID: tb-snmp-transport + env_file: + - tb-snmp-transport.env + volumes: + - ./tb-transports/snmp/conf:/config + - ./tb-transports/snmp/log:/var/log/tb-snmp-transport + depends_on: + - zookeeper tb-web-ui1: restart: always image: "${DOCKER_REPO}/${WEB_UI_DOCKER_NAME}:${TB_VERSION}" diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 3ef3292f22..e0a23715a3 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -24,3 +24,5 @@ mkdir -p tb-transports/lwm2m/log && sudo chown -R 799:799 tb-transports/lwm2m/lo mkdir -p tb-transports/http/log && sudo chown -R 799:799 tb-transports/http/log mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log + +mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log diff --git a/docker/tb-snmp-transport.env b/docker/tb-snmp-transport.env new file mode 100644 index 0000000000..f92e30b78f --- /dev/null +++ b/docker/tb-snmp-transport.env @@ -0,0 +1,2 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 3b1871b12b..6308b603c3 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -32,6 +32,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume"; private final static String TB_HTTP_TRANSPORT_LOG_VOLUME = "tb-http-transport-log-test-volume"; private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; + private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; private final DockerComposeExecutor dockerCompose; @@ -41,6 +42,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final String tbLwm2mTransportLogVolume; private final String tbHttpTransportLogVolume; private final String tbMqttTransportLogVolume; + private final String tbSnmpTransportLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -57,6 +59,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME; tbHttpTransportLogVolume = project + "_" + TB_HTTP_TRANSPORT_LOG_VOLUME; tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; + tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -67,6 +70,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_LWM2M_TRANSPORT_LOG_VOLUME", tbLwm2mTransportLogVolume); env.put("TB_HTTP_TRANSPORT_LOG_VOLUME", tbHttpTransportLogVolume); env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); + env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); dockerCompose.withEnv(env); } @@ -96,6 +100,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbMqttTransportLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbSnmpTransportLogVolume); + dockerCompose.invokeDocker(); + dockerCompose.withCommand("up -d redis postgres"); dockerCompose.invokeCompose(); @@ -117,9 +124,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { copyLogs(tbLwm2mTransportLogVolume, "./target/tb-lwm2m-transport-logs/"); copyLogs(tbHttpTransportLogVolume, "./target/tb-http-transport-logs/"); copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); + copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + - " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + " " + tbMqttTransportLogVolume); + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume); dockerCompose.invokeDocker(); } diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index db9f79b51a..ae524d5047 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -14,13 +14,6 @@ # limitations under the License. # -server: - # Server bind address - address: "${HTTP_BIND_ADDRESS:0.0.0.0}" - # Server bind port - port: "${HTTP_BIND_PORT:8085}" - # Server SSL configuration - # Zookeeper connection parameters. Used for service discovery. zk: # Enable/disable zookeeper discovery service. @@ -36,146 +29,14 @@ zk: # Name of the directory in zookeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" -cluster: - stats: - enabled: "${TB_CLUSTER_STATS_ENABLED:false}" - print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" - -cache: - # caffeine or redis - type: "${CACHE_TYPE:caffeine}" - attributes: - # make sure that if cache.type is 'redis' and cache.attributes.enabled is 'true' that you change 'maxmemory-policy' Redis config property to 'allkeys-lru', 'allkeys-lfu' or 'allkeys-random' - enabled: "${CACHE_ATTRIBUTES_ENABLED:true}" - -caffeine: - specs: - relations: - timeToLiveInMinutes: 1440 - maxSize: 0 - deviceCredentials: - timeToLiveInMinutes: 1440 - maxSize: 0 - devices: - timeToLiveInMinutes: 1440 - maxSize: 0 - sessions: - timeToLiveInMinutes: 1440 - maxSize: 0 - assets: - timeToLiveInMinutes: 1440 - maxSize: 0 - entityViews: - timeToLiveInMinutes: 1440 - maxSize: 0 - claimDevices: - timeToLiveInMinutes: 1 - maxSize: 0 - securitySettings: - timeToLiveInMinutes: 1440 - maxSize: 0 - tenantProfiles: - timeToLiveInMinutes: 1440 - maxSize: 0 - deviceProfiles: - timeToLiveInMinutes: 1440 - maxSize: 0 - attributes: - timeToLiveInMinutes: 1440 - maxSize: 100000 - -redis: - # standalone or cluster - connection: - type: "${REDIS_CONNECTION_TYPE:standalone}" - standalone: - host: "${REDIS_HOST:localhost}" - port: "${REDIS_PORT:6379}" - useDefaultClientConfig: "${REDIS_USE_DEFAULT_CLIENT_CONFIG:true}" - # this value may be used only if you used not default ClientConfig - clientName: "${REDIS_CLIENT_NAME:standalone}" - # this value may be used only if you used not default ClientConfig - connectTimeout: "${REDIS_CLIENT_CONNECT_TIMEOUT:30000}" - # this value may be used only if you used not default ClientConfig - readTimeout: "${REDIS_CLIENT_READ_TIMEOUT:60000}" - # this value may be used only if you used not default ClientConfig - usePoolConfig: "${REDIS_CLIENT_USE_POOL_CONFIG:false}" - cluster: - # Comma-separated list of "host:port" pairs to bootstrap from. - nodes: "${REDIS_NODES:}" - # Maximum number of redirects to follow when executing commands across the cluster. - max-redirects: "${REDIS_MAX_REDIRECTS:12}" - useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index - db: "${REDIS_DB:0}" - # db password - password: "${REDIS_PASSWORD:}" - # pool config - pool_config: - maxTotal: "${REDIS_POOL_CONFIG_MAX_TOTAL:128}" - maxIdle: "${REDIS_POOL_CONFIG_MAX_IDLE:128}" - minIdle: "${REDIS_POOL_CONFIG_MIN_IDLE:16}" - testOnBorrow: "${REDIS_POOL_CONFIG_TEST_ON_BORROW:true}" - testOnReturn: "${REDIS_POOL_CONFIG_TEST_ON_RETURN:true}" - testWhileIdle: "${REDIS_POOL_CONFIG_TEST_WHILE_IDLE:true}" - minEvictableMs: "${REDIS_POOL_CONFIG_MIN_EVICTABLE_MS:60000}" - evictionRunsMs: "${REDIS_POOL_CONFIG_EVICTION_RUNS_MS:30000}" - maxWaitMills: "${REDIS_POOL_CONFIG_MAX_WAIT_MS:60000}" - numberTestsPerEvictionRun: "${REDIS_POOL_CONFIG_NUMBER_TESTS_PER_EVICTION_RUN:3}" - blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" - -# Check new version updates parameters -updates: - # Enable/disable updates checking. - enabled: "${UPDATES_ENABLED:true}" - -# spring freemarker configuration -spring.freemarker.checkTemplateLocation: "false" - -audit-log: - # Enable/disable audit log functionality. - enabled: "${AUDIT_LOG_ENABLED:true}" - # Specify partitioning size for audit log by tenant id storage. Example MINUTES, HOURS, DAYS, MONTHS - by_tenant_partitioning: "${AUDIT_LOG_BY_TENANT_PARTITIONING:MONTHS}" - # Number of days as history period if startTime and endTime are not specified - default_query_period: "${AUDIT_LOG_DEFAULT_QUERY_PERIOD:30}" - # Logging levels per each entity type. - # Allowed values: OFF (disable), W (log write operations), RW (log read and write operations) - logging-level: - mask: - "device": "${AUDIT_LOG_MASK_DEVICE:W}" - "asset": "${AUDIT_LOG_MASK_ASSET:W}" - "dashboard": "${AUDIT_LOG_MASK_DASHBOARD:W}" - "customer": "${AUDIT_LOG_MASK_CUSTOMER:W}" - "user": "${AUDIT_LOG_MASK_USER:W}" - "rule_chain": "${AUDIT_LOG_MASK_RULE_CHAIN:W}" - "alarm": "${AUDIT_LOG_MASK_ALARM:W}" - "entity_view": "${AUDIT_LOG_MASK_ENTITY_VIEW:W}" - "device_profile": "${AUDIT_LOG_MASK_DEVICE_PROFILE:W}" - sink: - # Type of external sink. possible options: none, elasticsearch - type: "${AUDIT_LOG_SINK_TYPE:none}" - # Name of the index where audit logs stored - # Index name could contain next placeholders (not mandatory): - # @{TENANT} - substituted by tenant ID - # @{DATE} - substituted by current date in format provided in audit_log.sink.date_format - index_pattern: "${AUDIT_LOG_SINK_INDEX_PATTERN:@{TENANT}_AUDIT_LOG_@{DATE}}" - # Date format. Details of the pattern could be found here: - # https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html - date_format: "${AUDIT_LOG_SINK_DATE_FORMAT:YYYY.MM.DD}" - scheme_name: "${AUDIT_LOG_SINK_SCHEME_NAME:http}" # http or https - host: "${AUDIT_LOG_SINK_HOST:localhost}" - port: "${AUDIT_LOG_SINK_PORT:9200}" - user_name: "${AUDIT_LOG_SINK_USER_NAME:}" - password: "${AUDIT_LOG_SINK_PASSWORD:}" - -state: - # Should be greater then transport.sessions.report_timeout - defaultInactivityTimeoutInSec: "${DEFAULT_INACTIVITY_TIMEOUT:600}" - defaultStateCheckIntervalInSec: "${DEFAULT_STATE_CHECK_INTERVAL:60}" - persistToTelemetry: "${PERSIST_STATE_TO_TELEMETRY:false}" - transport: + snmp: + enabled: "${SNMP_ENABLED:true}" + response_processing: + # parallelism level for executor (workStealingPool) that is responsible for handling responses from SNMP devices + parallelism_level: "${SNMP_RESPONSE_PROCESSING_PARALLELISM_LEVEL:20}" + # to configure SNMP to work over UDP or TCP + underlying_protocol: "${SNMP_UNDERLYING_PROTOCOL:udp}" sessions: inactivity_timeout: "${TB_TRANSPORT_SESSIONS_INACTIVITY_TIMEOUT:300000}" report_timeout: "${TB_TRANSPORT_SESSIONS_REPORT_TIMEOUT:30000}" @@ -184,23 +45,9 @@ transport: type_cast_enabled: "${JSON_TYPE_CAST_ENABLED:true}" # Maximum allowed string value length when processing Telemetry/Attributes JSON (0 value disables string value length check) max_string_value_length: "${JSON_MAX_STRING_VALUE_LENGTH:0}" - # Enable/disable http/mqtt/coap transport protocols (has higher priority than certain protocol's 'enabled' property) - api_enabled: "${TB_TRANSPORT_API_ENABLED:true}" - # Local LwM2M transport parameters - snmp: - enabled: "${SNMP_ENABLED:true}" - response_processing: - # parallelism level for executor (workStealingPool) that is responsible for handling responses from SNMP devices - parallelism_level: "${SNMP_RESPONSE_PROCESSING_PARALLELISM_LEVEL:20}" - # to configure SNMP to work over UDP or TCP - underlying_protocol: "${SNMP_UNDERLYING_PROTOCOL:udp}" queue: - type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) - in_memory: - stats: - # For debug lvl - print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" + type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" @@ -209,10 +56,6 @@ queue: linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" - max_poll_interval_ms: "${TB_QUEUE_KAFKA_MAX_POLL_INTERVAL_MS:300000}" - max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" - max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" - fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" confluent: ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" @@ -221,11 +64,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" consumer-stats: enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" @@ -282,7 +125,8 @@ queue: notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: - hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 + hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" + virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" transport_api: requests_topic: "${TB_QUEUE_TRANSPORT_API_REQUEST_TOPIC:tb_transport.api.requests}" responses_topic: "${TB_QUEUE_TRANSPORT_API_RESPONSE_TOPIC:tb_transport.api.responses}" @@ -295,11 +139,11 @@ queue: topic: "${TB_QUEUE_CORE_TOPIC:tb_core}" poll-interval: "${TB_QUEUE_CORE_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_CORE_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_CORE_PACK_PROCESSING_TIMEOUT_MS:60000}" usage-stats-topic: "${TB_QUEUE_US_TOPIC:tb_usage_stats}" stats: - enabled: "${TB_QUEUE_CORE_STATS_ENABLED:true}" - print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:60000}" + enabled: "${TB_QUEUE_CORE_STATS_ENABLED:false}" + print-interval-ms: "${TB_QUEUE_CORE_STATS_PRINT_INTERVAL_MS:10000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" @@ -308,15 +152,13 @@ queue: # JS Eval max pending requests max_pending_requests: "${REMOTE_JS_MAX_PENDING_REQUESTS:10000}" # JS Eval max request timeout - max_eval_requests_timeout: "${REMOTE_JS_MAX_EVAL_REQUEST_TIMEOUT:60000}" - # JS max request timeout max_requests_timeout: "${REMOTE_JS_MAX_REQUEST_TIMEOUT:10000}" # JS response poll interval response_poll_interval: "${REMOTE_JS_RESPONSE_POLL_INTERVAL_MS:25}" rule-engine: topic: "${TB_QUEUE_RULE_ENGINE_TOPIC:tb_rule_engine}" poll-interval: "${TB_QUEUE_RULE_ENGINE_POLL_INTERVAL_MS:25}" - pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_RULE_ENGINE_PACK_PROCESSING_TIMEOUT_MS:60000}" stats: enabled: "${TB_QUEUE_RULE_ENGINE_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_RULE_ENGINE_STATS_PRINT_INTERVAL_MS:60000}" @@ -325,7 +167,7 @@ queue: topic: "${TB_QUEUE_RE_MAIN_TOPIC:tb_rule_engine.main}" poll-interval: "${TB_QUEUE_RE_MAIN_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_MAIN_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_RE_MAIN_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_MAIN_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only @@ -336,12 +178,11 @@ queue: retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_RETRY_PAUSE:3}"# Time in seconds to wait in consumer thread before retries; - max-pause-between-retries: "${TB_QUEUE_RE_MAIN_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:3}"# Max allowed time in seconds for pause between retries. - name: "${TB_QUEUE_RE_HP_QUEUE_NAME:HighPriority}" topic: "${TB_QUEUE_RE_HP_TOPIC:tb_rule_engine.hp}" poll-interval: "${TB_QUEUE_RE_HP_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_HP_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_RE_HP_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_HP_SUBMIT_STRATEGY_TYPE:BURST}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only @@ -352,12 +193,11 @@ queue: retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRIES:0}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - max-pause-between-retries: "${TB_QUEUE_RE_HP_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries. - name: "${TB_QUEUE_RE_SQ_QUEUE_NAME:SequentialByOriginator}" topic: "${TB_QUEUE_RE_SQ_TOPIC:tb_rule_engine.sq}" poll-interval: "${TB_QUEUE_RE_SQ_POLL_INTERVAL_MS:25}" partitions: "${TB_QUEUE_RE_SQ_PARTITIONS:10}" - pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:2000}" + pack-processing-timeout: "${TB_QUEUE_RE_SQ_PACK_PROCESSING_TIMEOUT_MS:60000}" submit-strategy: type: "${TB_QUEUE_RE_SQ_SUBMIT_STRATEGY_TYPE:SEQUENTIAL_BY_ORIGINATOR}" # BURST, BATCH, SEQUENTIAL_BY_ORIGINATOR, SEQUENTIAL_BY_TENANT, SEQUENTIAL # For BATCH only @@ -368,16 +208,11 @@ queue: retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRIES:3}" # Number of retries, 0 is unlimited failure-percentage: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_FAILURE_PERCENTAGE:0}" # Skip retry if failures or timeouts are less then X percentage of messages; pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_RETRY_PAUSE:5}"# Time in seconds to wait in consumer thread before retries; - max-pause-between-retries: "${TB_QUEUE_RE_SQ_PROCESSING_STRATEGY_MAX_RETRY_PAUSE:5}"# Max allowed time in seconds for pause between retries. transport: # For high priority notifications that require minimum latency and processing time notifications_topic: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_TOPIC:tb_transport.notifications}" poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" -event: - debug: - max-symbols: "${TB_MAX_DEBUG_EVENT_SYMBOLS:4096}" - service: type: "${TB_SERVICE_TYPE:tb-transport}" # Unique id for this service (autogenerated if empty) @@ -387,13 +222,10 @@ service: metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" - timer: - # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). - percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" management: endpoints: web: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). - include: '${METRICS_ENDPOINTS_EXPOSE:info}' + include: '${METRICS_ENDPOINTS_EXPOSE:info}' \ No newline at end of file From 5976d5172969e0fb11f604686f75bb27a0a7a3eb Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Wed, 28 Apr 2021 14:48:17 +0300 Subject: [PATCH 15/32] UI: Add import/export tenant profile --- .../import-export/import-export.service.ts | 51 +++++++++++++++++-- .../tenant-profiles-table-config.resolver.ts | 48 ++++++++++++++++- .../assets/locale/locale.constant-en_US.json | 6 +++ 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/import-export/import-export.service.ts b/ui-ngx/src/app/modules/home/components/import-export/import-export.service.ts index 72b5e661e3..3bd63b046b 100644 --- a/ui-ngx/src/app/modules/home/components/import-export/import-export.service.ts +++ b/ui-ngx/src/app/modules/home/components/import-export/import-export.service.ts @@ -57,6 +57,8 @@ import { RuleChainService } from '@core/http/rule-chain.service'; import { FiltersInfo } from '@shared/models/query/query.models'; import { DeviceProfileService } from '@core/http/device-profile.service'; import { DeviceProfile } from '@shared/models/device.models'; +import { TenantProfile } from '@shared/models/tenant.model'; +import { TenantProfileService } from '@core/http/tenant-profile.service'; // @dynamic @Injectable() @@ -70,6 +72,7 @@ export class ImportExportService { private dashboardUtils: DashboardUtilsService, private widgetService: WidgetService, private deviceProfileService: DeviceProfileService, + private tenantProfileService: TenantProfileService, private entityService: EntityService, private ruleChainService: RuleChainService, private utils: UtilsService, @@ -434,7 +437,7 @@ export class ImportExportService { (deviceProfile) => { let name = deviceProfile.name; name = name.toLowerCase().replace(/\W/g, '_'); - this.exportToPc(this.prepareDeviceProfileExport(deviceProfile), name); + this.exportToPc(this.prepareProfileExport(deviceProfile), name); }, (e) => { this.handleExportError(e, 'device-profile.export-failed-error'); @@ -460,6 +463,37 @@ export class ImportExportService { ); } + public exportTenantProfile(tenantProfileId: string) { + this.tenantProfileService.getTenantProfile(tenantProfileId).subscribe( + (tenantProfile) => { + let name = tenantProfile.name; + name = name.toLowerCase().replace(/\W/g, '_'); + this.exportToPc(this.prepareProfileExport(tenantProfile), name); + }, + (e) => { + this.handleExportError(e, 'tenant-profile.export-failed-error'); + } + ); + } + + public importTenantProfile(): Observable { + return this.openImportDialog('tenant-profile.import', 'tenant-profile.tenant-profile-file').pipe( + mergeMap((tenantProfile: TenantProfile) => { + if (!this.validateImportedTenantProfile(tenantProfile)) { + this.store.dispatch(new ActionNotificationShow( + {message: this.translate.instant('tenant-profile.invalid-tenant-profile-file-error'), + type: 'error'})); + throw new Error('Invalid tenant profile file'); + } else { + return this.tenantProfileService.saveTenantProfile(tenantProfile); + } + }), + catchError(() => { + return of(null); + }) + ); + } + public exportJSZip(data: object, filename: string) { import('jszip').then((JSZip) => { const jsZip = new JSZip.default(); @@ -517,6 +551,13 @@ export class ImportExportService { return true; } + private validateImportedTenantProfile(tenantProfile: TenantProfile): boolean { + return isDefined(tenantProfile.name) + && isDefined(tenantProfile.profileData) + && isDefined(tenantProfile.isolatedTbCore) + && isDefined(tenantProfile.isolatedTbRuleEngine); + } + private sumObject(obj1: any, obj2: any): any { Object.keys(obj2).map((key) => { if (isObject(obj2[key])) { @@ -798,10 +839,10 @@ export class ImportExportService { return dashboard; } - private prepareDeviceProfileExport(deviceProfile: DeviceProfile): DeviceProfile { - deviceProfile = this.prepareExport(deviceProfile); - deviceProfile.default = false; - return deviceProfile; + private prepareProfileExport(profile: T): T { + profile = this.prepareExport(profile); + profile.default = false; + return profile; } private prepareExport(data: any): any { diff --git a/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts b/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts index 996058e753..6c280c096b 100644 --- a/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts +++ b/ui-ngx/src/app/modules/home/pages/tenant-profile/tenant-profiles-table-config.resolver.ts @@ -21,7 +21,8 @@ import { checkBoxCell, DateEntityTableColumn, EntityTableColumn, - EntityTableConfig + EntityTableConfig, + HeaderActionDescriptor } from '@home/models/entity/entities-table-config.models'; import { TranslateService } from '@ngx-translate/core'; import { DatePipe } from '@angular/common'; @@ -31,6 +32,7 @@ import { TenantProfileService } from '@core/http/tenant-profile.service'; import { TenantProfileComponent } from '../../components/profile/tenant-profile.component'; import { TenantProfileTabsComponent } from './tenant-profile-tabs.component'; import { DialogService } from '@core/services/dialog.service'; +import { ImportExportService } from '@home/components/import-export/import-export.service'; @Injectable() export class TenantProfilesTableConfigResolver implements Resolve> { @@ -38,6 +40,7 @@ export class TenantProfilesTableConfigResolver implements Resolve = new EntityTableConfig(); constructor(private tenantProfileService: TenantProfileService, + private importExport: ImportExportService, private translate: TranslateService, private datePipe: DatePipe, private dialogService: DialogService) { @@ -59,6 +62,12 @@ export class TenantProfilesTableConfigResolver implements Resolve true, + onAction: ($event, entity) => this.exportTenantProfile($event, entity) + }, { name: this.translate.instant('tenant-profile.set-default'), icon: 'flag', @@ -80,6 +89,7 @@ export class TenantProfilesTableConfigResolver implements Resolve this.onTenantProfileAction(action); this.config.deleteEnabled = (tenantProfile) => tenantProfile && !tenantProfile.default; this.config.entitySelectionEnabled = (tenantProfile) => tenantProfile && !tenantProfile.default; + this.config.addActionDescriptors = this.configureAddActions(); } resolve(): EntityTableConfig { @@ -88,6 +98,25 @@ export class TenantProfilesTableConfigResolver implements Resolve { + const actions: Array = []; + actions.push( + { + name: this.translate.instant('tenant-profile.create-tenant-profile'), + icon: 'insert_drive_file', + isEnabled: () => true, + onAction: ($event) => this.config.table.addEntity($event) + }, + { + name: this.translate.instant('tenant-profile.import'), + icon: 'file_upload', + isEnabled: () => true, + onAction: ($event) => this.importTenantProfile($event) + } + ); + return actions; + } + setDefaultTenantProfile($event: Event, tenantProfile: TenantProfile) { if ($event) { $event.stopPropagation(); @@ -110,6 +139,23 @@ export class TenantProfilesTableConfigResolver implements Resolve { + if (deviceProfile) { + this.config.table.updateData(); + } + } + ); + } + + exportTenantProfile($event: Event, tenantProfile: TenantProfile) { + if ($event) { + $event.stopPropagation(); + } + this.importExport.exportTenantProfile(tenantProfile.id.id); + } + onTenantProfileAction(action: EntityAction): boolean { switch (action.action) { case 'setDefault': diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index d59ee56236..c7de3c2a1c 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2440,6 +2440,12 @@ "set-default-tenant-profile-text": "After the confirmation the tenant profile will be marked as default and will be used for new tenants with no profile specified.", "no-tenant-profiles-found": "No tenant profiles found.", "create-new-tenant-profile": "Create a new one!", + "create-tenant-profile": "Create new tenant profile", + "import": "Import tenant profile", + "export": "Export tenant profile", + "export-failed-error": "Unable to export tenant profile: {{error}}", + "tenant-profile-file": "Tenant profile file", + "invalid-tenant-profile-file-error": "Unable to import tenant profile: Invalid tenant profile data structure.", "maximum-devices": "Maximum number of devices (0 - unlimited)", "maximum-devices-required": "Maximum number of devices is required.", "maximum-devices-range": "Minimum number of devices can't be negative", From 79eb1183d92ac7d03a09876dcbc6bc75240d1026 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 27 Apr 2021 15:14:35 +0300 Subject: [PATCH 16/32] Request SNMP devices' ids with pagination --- .../transport/DefaultTransportApiService.java | 15 +++++----- .../server/dao/device/DeviceService.java | 2 +- common/queue/src/main/proto/queue.proto | 4 ++- .../service/ProtoTransportEntityService.java | 30 +++++++++++++++---- .../server/dao/device/DeviceDao.java | 3 +- .../server/dao/device/DeviceServiceImpl.java | 6 ++-- .../dao/sql/device/DeviceRepository.java | 2 +- .../server/dao/sql/device/JpaDeviceDao.java | 5 ++-- 8 files changed, 42 insertions(+), 25 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index dff59985b9..826e1d84a3 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; -import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.JacksonUtil; @@ -46,6 +45,8 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; @@ -62,7 +63,6 @@ import org.thingsboard.server.dao.device.provision.ProvisionRequest; import org.thingsboard.server.dao.device.provision.ProvisionResponse; import org.thingsboard.server.dao.firmware.FirmwareService; import org.thingsboard.server.dao.relation.RelationService; -import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; @@ -72,9 +72,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequ import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetResourceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetSnmpDevicesResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.GetResourceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; @@ -91,7 +91,6 @@ import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.resource.TbResourceService; import org.thingsboard.server.service.state.DeviceStateService; -import java.util.List; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -433,13 +432,15 @@ public class DefaultTransportApiService implements TransportApiService { return Futures.immediateFuture(TransportApiResponseMsg.newBuilder().setResourceResponseMsg(builder).build()); } - // TODO: request snmp devices with pagination private ListenableFuture handle(GetSnmpDevicesRequestMsg requestMsg) { - List result = deviceService.findDevicesIdsByDeviceProfileTransportType(DeviceTransportType.SNMP); + PageLink pageLink = new PageLink(requestMsg.getPageSize(), requestMsg.getPage()); + PageData result = deviceService.findDevicesIdsByDeviceProfileTransportType(DeviceTransportType.SNMP, pageLink); + GetSnmpDevicesResponseMsg responseMsg = GetSnmpDevicesResponseMsg.newBuilder() - .addAllIds(result.stream() + .addAllIds(result.getData().stream() .map(UUID::toString) .collect(Collectors.toList())) + .setHasNextPage(result.hasNext()) .build(); return Futures.immediateFuture(TransportApiResponseMsg.newBuilder() diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index a6a8e6d719..8e111fa33f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -95,7 +95,7 @@ public interface DeviceService { Device saveDevice(ProvisionRequest provisionRequest, DeviceProfile profile); - List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType); + PageData findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType, PageLink pageLink); Device assignDeviceToEdge(TenantId tenantId, DeviceId deviceId, EdgeId edgeId); diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 3800a5d082..340f9c1b05 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -268,11 +268,13 @@ message GetDeviceCredentialsResponseMsg { } message GetSnmpDevicesRequestMsg { - + int32 page = 1; + int32 pageSize = 2; } message GetSnmpDevicesResponseMsg { repeated string ids = 1; + bool hasNextPage = 2; } message EntityUpdateMsg { diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java index 1f639daa2a..957d610f91 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.transport.util.DataDecodingEncodingService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbSnmpTransportComponent; +import java.util.ArrayList; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -81,12 +82,29 @@ public class ProtoTransportEntityService { } public List getAllSnmpDevicesIds() { - TransportProtos.GetSnmpDevicesResponseMsg devicesIdsResponse = transportService.getSnmpDevicesIds( - TransportProtos.GetSnmpDevicesRequestMsg.getDefaultInstance() - ); + List result = new ArrayList<>(); + + int page = 0; + int pageSize = 512; + boolean hasNextPage = true; + + while (hasNextPage) { + TransportProtos.GetSnmpDevicesResponseMsg responseMsg = requestSnmpDevicesIds(page, pageSize); + result.addAll(responseMsg.getIdsList().stream() + .map(UUID::fromString) + .collect(Collectors.toList())); + hasNextPage = responseMsg.getHasNextPage(); + page++; + } + + return result; + } - return devicesIdsResponse.getIdsList().stream() - .map(UUID::fromString) - .collect(Collectors.toList()); + private TransportProtos.GetSnmpDevicesResponseMsg requestSnmpDevicesIds(int page, int pageSize) { + TransportProtos.GetSnmpDevicesRequestMsg requestMsg = TransportProtos.GetSnmpDevicesRequestMsg.newBuilder() + .setPage(page) + .setPageSize(pageSize) + .build(); + return transportService.getSnmpDevicesIds(requestMsg); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index d325be4ddb..a714d5ec53 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.TenantEntityDao; @@ -220,7 +219,7 @@ public interface DeviceDao extends Dao, TenantEntityDao { */ PageData findDevicesByTenantIdAndProfileId(UUID tenantId, UUID profileId, PageLink pageLink); - List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType); + PageData findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType, PageLink pageLink); /** * Find devices by tenantId, edgeId and page link. diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index b67c44caa4..02d8bd2b15 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -42,7 +42,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.device.DeviceSearchQuery; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; import org.thingsboard.server.common.data.device.data.CoapDeviceTransportConfiguration; @@ -62,7 +61,6 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.relation.RelationTypeGroup; @@ -580,8 +578,8 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe } @Override - public List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) { - return deviceDao.findDevicesIdsByDeviceProfileTransportType(transportType); + public PageData findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType, PageLink pageLink) { + return deviceDao.findDevicesIdsByDeviceProfileTransportType(transportType, pageLink); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index 33e19397a1..b6d723f812 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -215,5 +215,5 @@ public interface DeviceRepository extends PagingAndSortingRepository findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType); + Page findIdsByDeviceProfileTransportType(@Param("transportType") DeviceTransportType transportType, Pageable pageable); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index f6b6b651e5..1291dc09e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.device.DeviceDao; import org.thingsboard.server.dao.model.sql.DeviceEntity; @@ -119,8 +118,8 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao } @Override - public List findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType) { - return deviceRepository.findIdsByDeviceProfileTransportType(transportType); + public PageData findDevicesIdsByDeviceProfileTransportType(DeviceTransportType transportType, PageLink pageLink) { + return DaoUtil.pageToPageData(deviceRepository.findIdsByDeviceProfileTransportType(transportType, DaoUtil.toPageable(pageLink))); } @Override From b34198f30c6f852eb8fca0e9a37cd0fe2538add7 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 28 Apr 2021 16:39:13 +0300 Subject: [PATCH 17/32] refactored CoAP Server component (#4497) * refactored ce code base needed for coap-integration in pe * added anotation TbCoapServerComponent on CoapTransportService * license updated --- .../src/main/resources/thingsboard.yml | 4 +-- .../server/coapserver/CoapServerContext.java | 3 +-- .../coapserver/DefaultCoapServerService.java | 3 +-- .../server/coapserver/TbCoapDtlsSettings.java | 9 +++---- .../coapserver/TbCoapServerComponent.java | 26 +++++++++++++++++++ .../transport/coap/CoapTransportService.java | 3 ++- .../thingsboard/common/util/JacksonUtil.java | 10 +++++++ 7 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ca1654afa2..33760d174c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -621,9 +621,9 @@ transport: key_password: "${COAP_DTLS_KEY_PASSWORD:server_key_password}" # Key alias key_alias: "${COAP_DTLS_KEY_ALIAS:serveralias}" - # Skip certificate validity check for client certificates. - skip_validity_check_for_client_cert: "${COAP_DTLS_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}" x509: + # Skip certificate validity check for client certificates. + skip_validity_check_for_client_cert: "${TB_COAP_X509_DTLS_SKIP_VALIDITY_CHECK_FOR_CLIENT_CERT:false}" dtls_session_inactivity_timeout: "${TB_COAP_X509_DTLS_SESSION_INACTIVITY_TIMEOUT:86400000}" dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" # Local LwM2M transport parameters diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/CoapServerContext.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/CoapServerContext.java index 4129108ba3..0f5aae8bd8 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/CoapServerContext.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/CoapServerContext.java @@ -19,11 +19,10 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; @Slf4j -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')") +@TbCoapServerComponent @Component public class CoapServerContext { diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java index aebac1a86d..93f4e76551 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/DefaultCoapServerService.java @@ -23,7 +23,6 @@ import org.eclipse.californium.core.server.resources.Resource; import org.eclipse.californium.scandium.DTLSConnector; import org.eclipse.californium.scandium.config.DtlsConnectorConfig; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -39,7 +38,7 @@ import java.util.concurrent.TimeUnit; @Slf4j @Component -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')") +@TbCoapServerComponent public class DefaultCoapServerService implements CoapServerService { @Autowired diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java index 417a78da12..d5605ee29b 100644 --- a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapDtlsSettings.java @@ -39,7 +39,6 @@ import java.util.Collections; import java.util.Optional; @Slf4j -@ConditionalOnExpression("'${transport.coap.enabled}'=='true'") @ConditionalOnProperty(prefix = "transport.coap.dtls", value = "enabled", havingValue = "true", matchIfMissing = false) @Component public class TbCoapDtlsSettings { @@ -50,7 +49,7 @@ public class TbCoapDtlsSettings { @Value("${transport.coap.dtls.bind_port}") private Integer port; - @Value("${transport.coap.dtls.mode}") + @Value("${transport.coap.dtls.mode:NO_AUTH}") private String mode; @Value("${transport.coap.dtls.key_store}") @@ -65,13 +64,13 @@ public class TbCoapDtlsSettings { @Value("${transport.coap.dtls.key_alias}") private String keyAlias; - @Value("${transport.coap.dtls.skip_validity_check_for_client_cert}") + @Value("${transport.coap.dtls.x509.skip_validity_check_for_client_cert:false}") private boolean skipValidityCheckForClientCert; - @Value("${transport.coap.dtls.x509.dtls_session_inactivity_timeout}") + @Value("${transport.coap.dtls.x509.dtls_session_inactivity_timeout:86400000}") private long dtlsSessionInactivityTimeout; - @Value("${transport.coap.dtls.x509.dtls_session_report_timeout}") + @Value("${transport.coap.dtls.x509.dtls_session_report_timeout:1800000}") private long dtlsSessionReportTimeout; @Autowired diff --git a/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java new file mode 100644 index 0000000000..718af54ab6 --- /dev/null +++ b/common/coap-server/src/main/java/org/thingsboard/server/coapserver/TbCoapServerComponent.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.coapserver; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +@Retention(RetentionPolicy.RUNTIME) +@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')") +public @interface TbCoapServerComponent { +} diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java index db5261dd75..a739563858 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportService.java @@ -23,6 +23,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.TbTransportService; import org.thingsboard.server.coapserver.CoapServerService; +import org.thingsboard.server.coapserver.TbCoapServerComponent; import org.thingsboard.server.transport.coap.efento.CoapEfentoTransportResource; import javax.annotation.PostConstruct; @@ -30,7 +31,7 @@ import javax.annotation.PreDestroy; import java.net.UnknownHostException; @Service("CoapTransportService") -@ConditionalOnExpression("'${service.type:null}'=='tb-transport' || ('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true' && '${transport.coap.enabled}'=='true')") +@TbCoapServerComponent @Slf4j public class CoapTransportService implements TbTransportService { diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index 8d498fa853..aa250a7039 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; +import java.util.Arrays; /** * Created by Valerii Sosliuk on 5/12/2017. @@ -66,6 +67,15 @@ public class JacksonUtil { } } + public static JsonNode fromBytes(byte[] bytes) { + try { + return OBJECT_MAPPER.readTree(bytes); + } catch (IOException e) { + throw new IllegalArgumentException("The given byte[] value: " + + Arrays.toString(bytes) + " cannot be transformed to Json object", e); + } + } + public static String toString(Object value) { try { return value != null ? OBJECT_MAPPER.writeValueAsString(value) : null; From 3b74a806bc6c23d7319e561d7409fca1325a85c6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 28 Apr 2021 16:40:45 +0300 Subject: [PATCH 18/32] RPC functionality refactoring for SNMP (#4492) * Refactor RPC functionality for SNMP * Refactor * SNMP response PDU mapping refactoring --- .../snmp/config/SnmpCommunicationConfig.java | 4 ++- ...viceRpcRequestSnmpCommunicationConfig.java | 26 ++++++++++++++ .../transport/snmp/service/PduService.java | 2 +- .../snmp/service/SnmpTransportService.java | 35 ++++++++++++------- 4 files changed, 53 insertions(+), 14 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcRequestSnmpCommunicationConfig.java diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/SnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/SnmpCommunicationConfig.java index e2ba29abfa..0d673a4145 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/SnmpCommunicationConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/SnmpCommunicationConfig.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.transport.snmp.SnmpMethod; import org.thingsboard.server.common.data.transport.snmp.config.impl.ClientAttributesQueryingSnmpCommunicationConfig; import org.thingsboard.server.common.data.transport.snmp.config.impl.SharedAttributesSettingSnmpCommunicationConfig; import org.thingsboard.server.common.data.transport.snmp.config.impl.TelemetryQueryingSnmpCommunicationConfig; +import org.thingsboard.server.common.data.transport.snmp.config.impl.ToDeviceRpcRequestSnmpCommunicationConfig; import java.util.List; @@ -34,7 +35,8 @@ import java.util.List; @JsonSubTypes({ @Type(value = TelemetryQueryingSnmpCommunicationConfig.class, name = "TELEMETRY_QUERYING"), @Type(value = ClientAttributesQueryingSnmpCommunicationConfig.class, name = "CLIENT_ATTRIBUTES_QUERYING"), - @Type(value = SharedAttributesSettingSnmpCommunicationConfig.class, name = "SHARED_ATTRIBUTES_SETTING") + @Type(value = SharedAttributesSettingSnmpCommunicationConfig.class, name = "SHARED_ATTRIBUTES_SETTING"), + @Type(value = ToDeviceRpcRequestSnmpCommunicationConfig.class, name = "TO_DEVICE_RPC_REQUEST") }) public interface SnmpCommunicationConfig { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcRequestSnmpCommunicationConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcRequestSnmpCommunicationConfig.java new file mode 100644 index 0000000000..6683500445 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/transport/snmp/config/impl/ToDeviceRpcRequestSnmpCommunicationConfig.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.transport.snmp.config.impl; + +import org.thingsboard.server.common.data.transport.snmp.SnmpCommunicationSpec; +import org.thingsboard.server.common.data.transport.snmp.config.MultipleMappingsSnmpCommunicationConfig; + +public class ToDeviceRpcRequestSnmpCommunicationConfig extends MultipleMappingsSnmpCommunicationConfig { + @Override + public SnmpCommunicationSpec getSpec() { + return SnmpCommunicationSpec.TO_DEVICE_RPC_REQUEST; + } +} diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduService.java index 720b1e7ae3..2fc9057048 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/PduService.java @@ -151,7 +151,7 @@ public class PduService { .collect(Collectors.toMap(VariableBinding::getOid, VariableBinding::toValueString)); } - private void processValue(String key, DataType dataType, String value, JsonObject result) { + public void processValue(String key, DataType dataType, String value, JsonObject result) { switch (dataType) { case LONG: result.addProperty(key, Long.parseLong(value)); diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java index 5bfee14c41..3c296cc4ae 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/SnmpTransportService.java @@ -183,19 +183,28 @@ public class SnmpTransportService implements TbTransportService { SnmpMethod snmpMethod = SnmpMethod.valueOf(toDeviceRpcRequestMsg.getMethodName()); JsonObject params = JsonConverter.parse(toDeviceRpcRequestMsg.getParams()).getAsJsonObject(); - String oid = Optional.ofNullable(params.get("oid")).map(JsonElement::getAsString).orElse(null); + String key = Optional.ofNullable(params.get("key")).map(JsonElement::getAsString).orElse(null); String value = Optional.ofNullable(params.get("value")).map(JsonElement::getAsString).orElse(null); - DataType dataType = Optional.ofNullable(params.get("dataType")).map(e -> DataType.valueOf(e.getAsString())).orElse(DataType.STRING); - if (oid == null || oid.isEmpty()) { - throw new IllegalArgumentException("OID in to-device RPC request is not specified"); - } if (value == null && snmpMethod == SnmpMethod.SET) { throw new IllegalArgumentException("Value must be specified for SNMP method 'SET'"); } + SnmpCommunicationConfig communicationConfig = sessionContext.getProfileTransportConfiguration().getCommunicationConfigs().stream() + .filter(config -> config.getSpec() == SnmpCommunicationSpec.TO_DEVICE_RPC_REQUEST) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No communication config found with RPC spec")); + SnmpMapping snmpMapping = communicationConfig.getAllMappings().stream() + .filter(mapping -> mapping.getKey().equals(key)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No SNMP mapping found in the config for specified key")); + + String oid = snmpMapping.getOid(); + DataType dataType = snmpMapping.getDataType(); + PDU request = pduService.createSingleVariablePdu(sessionContext, snmpMethod, oid, value, dataType); - sendRequest(sessionContext, request, new RequestInfo(toDeviceRpcRequestMsg.getRequestId(), SnmpCommunicationSpec.TO_DEVICE_RPC_REQUEST)); + RequestInfo requestInfo = new RequestInfo(toDeviceRpcRequestMsg.getRequestId(), communicationConfig.getSpec(), communicationConfig.getAllMappings()); + sendRequest(sessionContext, request, requestInfo); } @@ -238,7 +247,12 @@ public class SnmpTransportService implements TbTransportService { responseDataMappers.put(SnmpCommunicationSpec.TO_DEVICE_RPC_REQUEST, (pdu, requestInfo) -> { JsonObject responseData = new JsonObject(); pduService.processPdu(pdu).forEach((oid, value) -> { - responseData.addProperty(oid.toDottedString(), value); + requestInfo.getResponseMappings().stream() + .filter(snmpMapping -> snmpMapping.getOid().equals(oid.toDottedString())) + .findFirst() + .ifPresent(snmpMapping -> { + pduService.processValue(snmpMapping.getKey(), snmpMapping.getDataType(), value, responseData); + }); }); return responseData; }); @@ -314,13 +328,10 @@ public class SnmpTransportService implements TbTransportService { private SnmpCommunicationSpec communicationSpec; private List responseMappings; - public RequestInfo(Integer requestId, SnmpCommunicationSpec communicationSpec) { + public RequestInfo(Integer requestId, SnmpCommunicationSpec communicationSpec, List responseMappings) { this.requestId = requestId; this.communicationSpec = communicationSpec; - } - - public RequestInfo(SnmpCommunicationSpec communicationSpec) { - this.communicationSpec = communicationSpec; + this.responseMappings = responseMappings; } public RequestInfo(SnmpCommunicationSpec communicationSpec, List responseMappings) { From 1de97ad0e1586d3917755cc257ff0bbf8d5e43e7 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 29 Apr 2021 11:12:53 +0300 Subject: [PATCH 19/32] Refactor SNMP devices' sessions establishing --- .../transport/snmp/SnmpTransportContext.java | 34 +++++++++++-------- .../service/ProtoTransportEntityService.java | 21 +----------- 2 files changed, 20 insertions(+), 35 deletions(-) diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java index 68d4dd3933..0efde4d893 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/SnmpTransportContext.java @@ -55,7 +55,6 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedDeque; -import java.util.stream.Collectors; @TbSnmpTransportComponent @Component @@ -72,25 +71,30 @@ public class SnmpTransportContext extends TransportContext { private final SnmpAuthService snmpAuthService; private final Map sessions = new ConcurrentHashMap<>(); - private Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); + private final Collection allSnmpDevicesIds = new ConcurrentLinkedDeque<>(); @AfterStartUp(order = 2) - public void initDevicesSessions() { + public void fetchDevicesAndEstablishSessions() { log.info("Initializing SNMP devices sessions"); - allSnmpDevicesIds = protoEntityService.getAllSnmpDevicesIds().stream() - .map(DeviceId::new) - .collect(Collectors.toList()); - log.trace("Found all SNMP devices ids: {}", allSnmpDevicesIds); - List managedDevicesIds = allSnmpDevicesIds.stream() - .filter(deviceId -> balancingService.isManagedByCurrentTransport(deviceId.getId())) - .collect(Collectors.toList()); - log.info("SNMP devices managed by current SNMP transport: {}", managedDevicesIds); + int batchIndex = 0; + int batchSize = 512; + boolean nextBatchExists = true; - managedDevicesIds.stream() - .map(protoEntityService::getDeviceById) - .collect(Collectors.toList()) - .forEach(this::establishDeviceSession); + while (nextBatchExists) { + TransportProtos.GetSnmpDevicesResponseMsg snmpDevicesResponse = protoEntityService.getSnmpDevicesIds(batchIndex, batchSize); + snmpDevicesResponse.getIdsList().stream() + .map(id -> new DeviceId(UUID.fromString(id))) + .peek(allSnmpDevicesIds::add) + .filter(deviceId -> balancingService.isManagedByCurrentTransport(deviceId.getId())) + .map(protoEntityService::getDeviceById) + .forEach(device -> getExecutor().execute(() -> establishDeviceSession(device))); + + nextBatchExists = snmpDevicesResponse.getHasNextPage(); + batchIndex++; + } + + log.debug("Found all SNMP devices ids: {}", allSnmpDevicesIds); } private void establishDeviceSession(Device device) { diff --git a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java index 957d610f91..777650f779 100644 --- a/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java +++ b/common/transport/snmp/src/main/java/org/thingsboard/server/transport/snmp/service/ProtoTransportEntityService.java @@ -81,26 +81,7 @@ public class ProtoTransportEntityService { .orElseThrow(() -> new IllegalArgumentException("Device credentials not found")); } - public List getAllSnmpDevicesIds() { - List result = new ArrayList<>(); - - int page = 0; - int pageSize = 512; - boolean hasNextPage = true; - - while (hasNextPage) { - TransportProtos.GetSnmpDevicesResponseMsg responseMsg = requestSnmpDevicesIds(page, pageSize); - result.addAll(responseMsg.getIdsList().stream() - .map(UUID::fromString) - .collect(Collectors.toList())); - hasNextPage = responseMsg.getHasNextPage(); - page++; - } - - return result; - } - - private TransportProtos.GetSnmpDevicesResponseMsg requestSnmpDevicesIds(int page, int pageSize) { + public TransportProtos.GetSnmpDevicesResponseMsg getSnmpDevicesIds(int page, int pageSize) { TransportProtos.GetSnmpDevicesRequestMsg requestMsg = TransportProtos.GetSnmpDevicesRequestMsg.newBuilder() .setPage(page) .setPageSize(pageSize) From b4b5d062b06a7d617cb1ace2c4480b2ed9972f84 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 23 Apr 2021 19:00:17 +0300 Subject: [PATCH 20/32] fixed alarm query (search without created time column) --- .../query/DefaultAlarmQueryRepository.java | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 5173b83fb8..a0ce183b97 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -17,12 +17,8 @@ package org.thingsboard.server.dao.sql.query; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate; import org.springframework.stereotype.Repository; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.alarm.AlarmSearchStatus; @@ -40,8 +36,6 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.dao.model.ModelConstants; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -134,6 +128,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { StringBuilder fromPart = new StringBuilder(" from alarm a "); StringBuilder wherePart = new StringBuilder(" where "); StringBuilder sortPart = new StringBuilder(" order by "); + StringBuilder joinPart = new StringBuilder(); boolean addAnd = false; if (pageLink.isSearchPropagatedAlarms()) { selectPart.append(" CASE WHEN r.from_id IS NULL THEN a.originator_id ELSE r.from_id END as entity_id "); @@ -156,23 +151,23 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { wherePart.append(" a.originator_id in (:entity_ids)"); } } else { - fromPart.append(" inner join (select * from (VALUES"); + joinPart.append(" inner join (select * from (VALUES"); int entityIdIdx = 0; int lastEntityIdIdx = orderedEntityIds.size() - 1; for (EntityId entityId : orderedEntityIds) { - fromPart.append("(uuid('").append(entityId.getId().toString()).append("'), ").append(entityIdIdx).append(")"); + joinPart.append("(uuid('").append(entityId.getId().toString()).append("'), ").append(entityIdIdx).append(")"); if (entityIdIdx != lastEntityIdIdx) { - fromPart.append(","); + joinPart.append(","); } else { - fromPart.append(")"); + joinPart.append(")"); } entityIdIdx++; } - fromPart.append(" as e(id, priority)) e "); + joinPart.append(" as e(id, priority)) e "); if (pageLink.isSearchPropagatedAlarms()) { - fromPart.append("on (r.from_id IS NULL and a.originator_id = e.id) or (r.from_id IS NOT NULL and r.from_id = e.id)"); + joinPart.append("on (r.from_id IS NULL and a.originator_id = e.id) or (r.from_id IS NOT NULL and r.from_id = e.id)"); } else { - fromPart.append("on a.originator_id = e.id"); + joinPart.append("on a.originator_id = e.id"); } sortPart.append("e.priority"); } @@ -226,9 +221,12 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch()); - String mainQuery = selectPart.toString() + fromPart.toString() + wherePart.toString(); + String mainQuery; if (!textSearchQuery.isEmpty()) { - mainQuery = String.format("select * from (%s) a WHERE %s", mainQuery, textSearchQuery); + mainQuery = selectPart.toString() + fromPart.toString() + wherePart.toString(); + mainQuery = String.format("select * from (%s) a %s WHERE %s", mainQuery, joinPart, textSearchQuery); + } else { + mainQuery = selectPart.toString() + fromPart.toString() + joinPart.toString() + wherePart.toString(); } String countQuery = String.format("select count(*) from (%s) result", mainQuery); long queryTs = System.currentTimeMillis(); From 39f07757664f450f7ef04152ecdc9398b0353828 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko <79898499+smatvienko-tb@users.noreply.github.com> Date: Thu, 29 Apr 2021 14:15:50 +0300 Subject: [PATCH 21/32] Thread pool naming CE (#4511) * added ThingsBoardExecutors and ThingsBoardForkJoinWorkerThreadFactory to define the name for the pool thread * thread poll named for the AbstractListeningExecutor * thread poll named for the RestClient. ThingsBoard util dependency added * most of thread polls named CE * thread poll name added for telemetry-web-socket-ping * executors: added custom names for executors (Queue, RuleEngine). Add topic name to the Thread name (useful for JMX and thread dump) * fixed licence header for a new classes --- .../actors/service/DefaultActorService.java | 3 +- .../DefaultTbApiUsageStateService.java | 3 +- .../DefaultTbRuleEngineConsumerService.java | 4 +- .../AbstractNashornJsInvokeService.java | 3 +- .../DefaultTbLocalSubscriptionService.java | 5 +- .../DefaultTelemetryWebSocketService.java | 6 ++- .../transport/TbCoreTransportApiService.java | 3 +- .../common/DefaultTbQueueRequestTemplate.java | 3 +- .../DefaultTbQueueResponseTemplate.java | 5 +- .../common/transport/TransportContext.java | 3 +- .../service/DefaultTransportService.java | 3 +- .../util/AbstractListeningExecutor.java | 2 +- .../common/util/ThingsBoardExecutors.java | 50 +++++++++++++++++++ ...hingsBoardForkJoinWorkerThreadFactory.java | 41 +++++++++++++++ .../util/AbstractBufferedRateExecutor.java | 3 +- rest-client/pom.xml | 4 ++ .../thingsboard/rest/client/RestClient.java | 3 +- 17 files changed, 127 insertions(+), 17 deletions(-) create mode 100644 common/util/src/main/java/org/thingsboard/common/util/ThingsBoardExecutors.java create mode 100644 common/util/src/main/java/org/thingsboard/common/util/ThingsBoardForkJoinWorkerThreadFactory.java diff --git a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java index f3a07eb727..7a28c205bc 100644 --- a/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java +++ b/application/src/main/java/org/thingsboard/server/actors/service/DefaultActorService.java @@ -22,6 +22,7 @@ import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.DefaultTbActorSystem; @@ -108,7 +109,7 @@ public class DefaultActorService extends TbApplicationEventListener tbRuleEngineQueueFactory.createToRuleEngineMsgConsumer(configuration)); consumerStats.put(configuration.getName(), new TbRuleEngineConsumerStats(configuration.getName(), statsFactory)); } - submitExecutor = Executors.newSingleThreadExecutor(); + submitExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("tb-rule-engine-consumer-service-submit-executor")); } @PreDestroy @@ -160,6 +161,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< private void launchConsumer(TbQueueConsumer> consumer, TbRuleEngineQueueConfiguration configuration, TbRuleEngineConsumerStats stats) { consumersExecutor.execute(() -> { + Thread.currentThread().setName("" + Thread.currentThread().getName() + "-" + configuration.getName()); while (!stopped) { try { List> msgs = consumer.poll(pollDuration); diff --git a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java index f581145907..9985ac60a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java +++ b/application/src/main/java/org/thingsboard/server/service/script/AbstractNashornJsInvokeService.java @@ -25,6 +25,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.queue.usagestats.TbApiUsageClient; import org.thingsboard.server.service.apiusage.TbApiUsageStateService; @@ -93,7 +94,7 @@ public abstract class AbstractNashornJsInvokeService extends AbstractJsInvokeSer super.init(maxRequestsTimeout); if (useJsSandbox()) { sandbox = NashornSandboxes.create(); - monitorExecutorService = Executors.newWorkStealingPool(getMonitorThreadPoolSize()); + monitorExecutorService = ThingsBoardExecutors.newWorkStealingPool(getMonitorThreadPoolSize(), "nashorn-js-monitor"); sandbox.setExecutor(monitorExecutorService); sandbox.setMaxCPUTime(getMaxCpuTime()); sandbox.allowNoBraces(false); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 710d613b37..0f1765f316 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.event.ClusterTopologyChangeEvent; import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; @@ -62,7 +63,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer private SubscriptionManagerService subscriptionManagerService; private ExecutorService subscriptionUpdateExecutor; - + private TbApplicationEventListener partitionChangeListener = new TbApplicationEventListener<>() { @Override protected void onTbApplicationEvent(PartitionChangeEvent event) { @@ -93,7 +94,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer @PostConstruct public void initExecutor() { - subscriptionUpdateExecutor = Executors.newWorkStealingPool(20); + subscriptionUpdateExecutor = ThingsBoardExecutors.newWorkStealingPool(20, getClass()); } @PreDestroy diff --git a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java index f8f4afd823..73eed820ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java +++ b/application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetryWebSocketService.java @@ -28,6 +28,8 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.springframework.web.socket.CloseStatus; +import org.thingsboard.common.util.ThingsBoardExecutors; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -157,9 +159,9 @@ public class DefaultTelemetryWebSocketService implements TelemetryWebSocketServi @PostConstruct public void initExecutor() { serviceId = serviceInfoProvider.getServiceId(); - executor = Executors.newWorkStealingPool(50); + executor = ThingsBoardExecutors.newWorkStealingPool(50, getClass()); - pingExecutor = Executors.newSingleThreadScheduledExecutor(); + pingExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("telemetry-web-socket-ping")); pingExecutor.scheduleWithFixedDelay(this::sendPing, 10000, 10000, TimeUnit.MILLISECONDS); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java index e9048d2d1c..dc6255cbc4 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/TbCoreTransportApiService.java @@ -21,6 +21,7 @@ import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.stats.MessagesStats; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.stats.StatsType; @@ -70,7 +71,7 @@ public class TbCoreTransportApiService { @PostConstruct public void init() { - this.transportCallbackExecutor = Executors.newWorkStealingPool(maxCallbackThreads); + this.transportCallbackExecutor = ThingsBoardExecutors.newWorkStealingPool(maxCallbackThreads, getClass()); TbQueueProducer> producer = tbCoreQueueFactory.createTransportApiResponseProducer(); TbQueueConsumer> consumer = tbCoreQueueFactory.createTransportApiRequestConsumer(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index 9562964fda..b171f2d8d8 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -20,6 +20,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.SettableFuture; import lombok.Builder; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.queue.TbQueueAdmin; import org.thingsboard.server.queue.TbQueueCallback; @@ -77,7 +78,7 @@ public class DefaultTbQueueRequestTemplate(queueLimit); this.dispatcherExecutor = Executors.newFixedThreadPool(dispatcherThreads, ThingsBoardThreadFactory.forName("nosql-dispatcher")); - this.callbackExecutor = Executors.newWorkStealingPool(callbackThreads); + this.callbackExecutor = ThingsBoardExecutors.newWorkStealingPool(callbackThreads, getClass()); this.timeoutExecutor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("nosql-timeout")); this.perTenantLimitsEnabled = perTenantLimitsEnabled; this.perTenantLimitsConfiguration = perTenantLimitsConfiguration; diff --git a/rest-client/pom.xml b/rest-client/pom.xml index e45db432d8..7272861a3b 100644 --- a/rest-client/pom.xml +++ b/rest-client/pom.xml @@ -43,6 +43,10 @@ org.springframework spring-web + + org.thingsboard.common + util + diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index e51117c080..433e2f8d96 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -31,6 +31,7 @@ import org.springframework.http.client.support.HttpRequestWrapper; import org.springframework.util.StringUtils; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; +import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rest.client.utils.RestJsonConverter; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.ClaimRequest; @@ -144,7 +145,7 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { private String token; private String refreshToken; private final ObjectMapper objectMapper = new ObjectMapper(); - private ExecutorService service = Executors.newWorkStealingPool(10); + private ExecutorService service = ThingsBoardExecutors.newWorkStealingPool(10, getClass()); protected static final String ACTIVATE_TOKEN_REGEX = "/api/noauth/activate?activateToken="; From 42fb44ff31efaf106a6981537805873c26aa9ffb Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 27 Apr 2021 13:07:21 +0300 Subject: [PATCH 22/32] concurrency test fix TbMsgPackProcessingContextTest (readyLatch). any logs commented to not affect concurrency with sync logger. refactored --- .../queue/TbMsgPackProcessingContextTest.java | 71 ++++++++++++++----- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java index 6c4e4a9426..1659431540 100644 --- a/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java +++ b/application/src/test/java/org/thingsboard/server/service/queue/TbMsgPackProcessingContextTest.java @@ -16,10 +16,10 @@ package org.thingsboard.server.service.queue; import lombok.extern.slf4j.Slf4j; +import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -28,39 +28,74 @@ import org.thingsboard.server.service.queue.processing.TbRuleEngineSubmitStrateg import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @Slf4j @RunWith(MockitoJUnitRunner.class) public class TbMsgPackProcessingContextTest { + public static final int TIMEOUT = 10; + ExecutorService executorService; + + @After + public void tearDown() { + if (executorService != null) { + executorService.shutdownNow(); + } + } + @Test public void testHighConcurrencyCase() throws InterruptedException { - TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); + //log.warn("preparing the test..."); int msgCount = 1000; int parallelCount = 5; - ExecutorService executorService = Executors.newFixedThreadPool(parallelCount); - try { - ConcurrentMap> messages = new ConcurrentHashMap<>(); - for (int i = 0; i < msgCount; i++) { - messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); - } - when(strategyMock.getPendingMap()).thenReturn(messages); - TbMsgPackProcessingContext context = new TbMsgPackProcessingContext("Main", strategyMock); - for (UUID uuid : messages.keySet()) { - for (int i = 0; i < parallelCount; i++) { - executorService.submit(() -> context.onSuccess(uuid)); - } + executorService = Executors.newFixedThreadPool(parallelCount); + + ConcurrentMap> messages = new ConcurrentHashMap<>(msgCount); + for (int i = 0; i < msgCount; i++) { + messages.put(UUID.randomUUID(), new TbProtoQueueMsg<>(UUID.randomUUID(), null)); + } + TbRuleEngineSubmitStrategy strategyMock = mock(TbRuleEngineSubmitStrategy.class); + when(strategyMock.getPendingMap()).thenReturn(messages); + + TbMsgPackProcessingContext context = new TbMsgPackProcessingContext("Main", strategyMock); + for (UUID uuid : messages.keySet()) { + final CountDownLatch readyLatch = new CountDownLatch(parallelCount); + final CountDownLatch startLatch = new CountDownLatch(1); + final CountDownLatch finishLatch = new CountDownLatch(parallelCount); + for (int i = 0; i < parallelCount; i++) { + //final String taskName = "" + uuid + " " + i; + executorService.submit(() -> { + //log.warn("ready {}", taskName); + readyLatch.countDown(); + try { + startLatch.await(); + } catch (InterruptedException e) { + Assert.fail("failed to await"); + } + //log.warn("go {}", taskName); + + context.onSuccess(uuid); + + finishLatch.countDown(); + }); } - Assert.assertTrue(context.await(10, TimeUnit.SECONDS)); - Mockito.verify(strategyMock, Mockito.times(msgCount)).onSuccess(Mockito.any(UUID.class)); - } finally { - executorService.shutdownNow(); + assertTrue(readyLatch.await(TIMEOUT, TimeUnit.SECONDS)); + Thread.yield(); + startLatch.countDown(); //run all-at-once submitted tasks + assertTrue(finishLatch.await(TIMEOUT, TimeUnit.SECONDS)); } + assertTrue(context.await(TIMEOUT, TimeUnit.SECONDS)); + verify(strategyMock, times(msgCount)).onSuccess(any(UUID.class)); } } From 5ad113ad1a6476e4b291c3b31b8903d8a23a8fd7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 23 Apr 2021 14:29:00 +0300 Subject: [PATCH 23/32] queue consumer: not going to sleep after pull if time left less then 1 millisecond. topic added to logs --- .../common/AbstractTbQueueConsumerTemplate.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java index 4481e52461..fc3ddf22ff 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/AbstractTbQueueConsumerTemplate.java @@ -38,6 +38,7 @@ import static java.util.Collections.emptyList; @Slf4j public abstract class AbstractTbQueueConsumerTemplate implements TbQueueConsumer { + public static final long ONE_MILLISECOND_IN_NANOS = TimeUnit.MILLISECONDS.toNanos(1); private volatile boolean subscribed; protected volatile boolean stopped = false; protected volatile Set partitions; @@ -83,7 +84,7 @@ public abstract class AbstractTbQueueConsumerTemplate i } if (consumerLock.isLocked()) { - log.error("poll. consumerLock is locked. will wait with no timeout. it looks like a race conditions or deadlock", new RuntimeException("stacktrace")); + log.error("poll. consumerLock is locked. will wait with no timeout. it looks like a race conditions or deadlock topic " + topic, new RuntimeException("stacktrace")); } consumerLock.lock(); @@ -131,9 +132,12 @@ public abstract class AbstractTbQueueConsumerTemplate i List sleepAndReturnEmpty(final long startNanos, final long durationInMillis) { long durationNanos = TimeUnit.MILLISECONDS.toNanos(durationInMillis); long spentNanos = System.nanoTime() - startNanos; - if (spentNanos < durationNanos) { + long nanosLeft = durationNanos - spentNanos; + if (nanosLeft >= ONE_MILLISECOND_IN_NANOS) { try { - Thread.sleep(Math.max(TimeUnit.NANOSECONDS.toMillis(durationNanos - spentNanos), 1)); + long sleepMs = TimeUnit.NANOSECONDS.toMillis(nanosLeft); + log.trace("Going to sleep after poll: topic {} for {}ms", topic, sleepMs); + Thread.sleep(sleepMs); } catch (InterruptedException e) { if (!stopped) { log.error("Failed to wait", e); @@ -146,7 +150,7 @@ public abstract class AbstractTbQueueConsumerTemplate i @Override public void commit() { if (consumerLock.isLocked()) { - log.error("commit. consumerLock is locked. will wait with no timeout. it looks like a race conditions or deadlock", new RuntimeException("stacktrace")); + log.error("commit. consumerLock is locked. will wait with no timeout. it looks like a race conditions or deadlock topic " + topic, new RuntimeException("stacktrace")); } consumerLock.lock(); try { From c25964d7472b2e3dd870e24198ddab2c1daf5c8f Mon Sep 17 00:00:00 2001 From: vzikratyi Date: Wed, 28 Apr 2021 15:13:26 +0300 Subject: [PATCH 24/32] Added prometheus-grafana monitoring to Docker scripts --- docker/.env | 3 + docker/README.md | 10 +++ docker/compose-utils.sh | 12 +++ docker/docker-compose.prometheus-grafana.yml | 47 +++++++++++ docker/docker-remove-services.sh | 4 +- docker/docker-start-services.sh | 4 +- docker/docker-stop-services.sh | 4 +- docker/monitoring/grafana/config.monitoring | 2 + .../provisioning/dashboards/dashboard.yml | 27 ++++++ .../provisioning/datasources/datasource.yml | 66 +++++++++++++++ docker/monitoring/prometheus/prometheus.yml | 83 +++++++++++++++++++ docker/tb-coap-transport.env | 6 ++ docker/tb-http-transport.env | 3 + docker/tb-mqtt-transport.env | 6 ++ docker/tb-node.env | 3 + 15 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 docker/docker-compose.prometheus-grafana.yml create mode 100644 docker/monitoring/grafana/config.monitoring create mode 100644 docker/monitoring/grafana/provisioning/dashboards/dashboard.yml create mode 100644 docker/monitoring/grafana/provisioning/datasources/datasource.yml create mode 100644 docker/monitoring/prometheus/prometheus.yml diff --git a/docker/.env b/docker/.env index e63e1dbf8d..11e44fdde8 100644 --- a/docker/.env +++ b/docker/.env @@ -19,3 +19,6 @@ TB_VERSION=latest DATABASE=postgres LOAD_BALANCER_NAME=haproxy-certbot + +# If enabled Prometheus and Grafana containers are deployed along with other containers +MONITORING_ENABLED=false \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 9664edc48e..01f89ebb6c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -100,3 +100,13 @@ $ ./docker-start-services.sh Where: - `FROM_VERSION` - from which version upgrade should be started. See [Upgrade Instructions](https://thingsboard.io/docs/user-guide/install/upgrade-instructions) for valid `fromVersion` values. + + +## Monitoring + +If you want to enable monitoring with Prometheus and Grafana you need to set MONITORING_ENABLED environment variable to true. +After this Prometheus and Grafana containers will be deployed. You can reach Prometheus at `http://localhost:9090` and Grafana at `http://localhost:3000` (default login is `admin` and password `foobar`). +To change Grafana password you need to update `GF_SECURITY_ADMIN_PASSWORD` environment variable at `./monitoring/grafana/config.monitoring` file. +Dashboards are loaded from `./monitoring/grafana/provisioning/dashboards` directory. + +If you want to add new monitoring jobs for Prometheus update `./monitoring/prometheus/prometheus.yml` file. \ No newline at end of file diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 6afc5ea2c1..db10fcd046 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -61,6 +61,18 @@ function additionalComposeQueueArgs() { echo $ADDITIONAL_COMPOSE_QUEUE_ARGS } +function additionalComposeMonitoringArgs() { + source .env + + if [ "$MONITORING_ENABLED" = true ] + then + ADDITIONAL_COMPOSE_MONITORING_ARGS="-f docker-compose.prometheus-grafana.yml" + echo $ADDITIONAL_COMPOSE_MONITORING_ARGS + else + echo "" + fi +} + function additionalStartupServices() { source .env ADDITIONAL_STARTUP_SERVICES="" diff --git a/docker/docker-compose.prometheus-grafana.yml b/docker/docker-compose.prometheus-grafana.yml new file mode 100644 index 0000000000..4fd53c68ec --- /dev/null +++ b/docker/docker-compose.prometheus-grafana.yml @@ -0,0 +1,47 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +volumes: + prometheus_data: {} + grafana_data: {} + +services: + + prometheus: + image: prom/prometheus:v2.1.0 + volumes: + - ./monitoring/prometheus/:/etc/prometheus/ + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - 9090:9090 + restart: always + grafana: + image: grafana/grafana + user: "472" + depends_on: + - prometheus + ports: + - 3000:3000 + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning/:/etc/grafana/provisioning/ + env_file: + - monitoring/grafana/config.monitoring + restart: always \ No newline at end of file diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index b2937e6c37..cd7a648b80 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -23,4 +23,6 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS down -v +ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? + +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index a3cb339652..75f98897ee 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -23,4 +23,6 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d +ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? + +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index d6751a1836..95f7f696f5 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -23,4 +23,6 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS stop +ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? + +docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop diff --git a/docker/monitoring/grafana/config.monitoring b/docker/monitoring/grafana/config.monitoring new file mode 100644 index 0000000000..f12466bb05 --- /dev/null +++ b/docker/monitoring/grafana/config.monitoring @@ -0,0 +1,2 @@ +GF_SECURITY_ADMIN_PASSWORD=foobar +GF_USERS_ALLOW_SIGN_UP=false diff --git a/docker/monitoring/grafana/provisioning/dashboards/dashboard.yml b/docker/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 0000000000..17a0d20d15 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: 1 + +providers: +- name: 'Prometheus' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/docker/monitoring/grafana/provisioning/datasources/datasource.yml b/docker/monitoring/grafana/provisioning/datasources/datasource.yml new file mode 100644 index 0000000000..c57361a4fa --- /dev/null +++ b/docker/monitoring/grafana/provisioning/datasources/datasource.yml @@ -0,0 +1,66 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# config file version +apiVersion: 1 + +# list of datasources that should be deleted from the database +deleteDatasources: + - name: Prometheus + orgId: 1 + +# list of datasources to insert/update depending +# whats available in the database +datasources: + # name of the datasource. Required +- name: Prometheus + # datasource type. Required + type: prometheus + # access mode. direct or proxy. Required + access: proxy + # org id. will default to orgId 1 if not specified + orgId: 1 + # url + url: http://prometheus:9090 + # database password, if used + password: + # database user, if used + user: + # database name, if used + database: + # enable/disable basic auth + basicAuth: false + # basic auth username, if used + basicAuthUser: + # basic auth password, if used + basicAuthPassword: + # enable/disable with credentials headers + withCredentials: + # mark as default datasource. Max one per org + isDefault: true + # fields that will be converted to json and stored in json_data + jsonData: + graphiteVersion: "1.1" + tlsAuth: false + tlsAuthWithCACert: false + # json object of data that will be encrypted. + secureJsonData: + tlsCACert: "..." + tlsClientCert: "..." + tlsClientKey: "..." + version: 1 + # allow users to edit datasources from the UI. + editable: true diff --git a/docker/monitoring/prometheus/prometheus.yml b/docker/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000000..a3b4c54e9c --- /dev/null +++ b/docker/monitoring/prometheus/prometheus.yml @@ -0,0 +1,83 @@ +# +# Copyright © 2016-2021 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# my global config +global: + scrape_interval: 15s # By default, scrape targets every 15 seconds. + evaluation_interval: 15s # By default, scrape targets every 15 seconds. + # scrape_timeout is set to the global default (10s). + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'thingsboard' + + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + + - job_name: 'prometheus' + scrape_interval: 5s + static_configs: + - targets: ['localhost:9090'] + + - job_name: 'tb-core1' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-core1:8080' ] + + - job_name: 'tb-core2' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-core2:8080' ] + + - job_name: 'tb-rule-engine1' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-rule-engine1:8080' ] + + - job_name: 'tb-rule-engine2' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-rule-engine2:8080' ] + + - job_name: 'tb-mqtt-transport1' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-mqtt-transport1:8081' ] + + - job_name: 'tb-mqtt-transport2' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-mqtt-transport2:8081' ] + + - job_name: 'tb-http-transport1' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-http-transport1:8081' ] + - + - job_name: 'tb-http-transport2' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-http-transport2:8081' ] + + - job_name: 'tb-coap-transport' + metrics_path: /actuator/prometheus + static_configs: + - targets: [ 'tb-coap-transport:8081' ] + diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index 406367a192..77ec90c2db 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -4,3 +4,9 @@ ZOOKEEPER_URL=zookeeper:2181 COAP_BIND_ADDRESS=0.0.0.0 COAP_BIND_PORT=5683 COAP_TIMEOUT=10000 + +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus +WEB_APPLICATION_ENABLE=true +WEB_APPLICATION_TYPE=servlet +HTTP_BIND_PORT=8081 \ No newline at end of file diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 41fb76453d..ab49d9b326 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -4,3 +4,6 @@ ZOOKEEPER_URL=zookeeper:2181 HTTP_BIND_ADDRESS=0.0.0.0 HTTP_BIND_PORT=8081 HTTP_REQUEST_TIMEOUT=60000 + +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus \ No newline at end of file diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 04f6ed8e98..a0927d9a2f 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -4,3 +4,9 @@ ZOOKEEPER_URL=zookeeper:2181 MQTT_BIND_ADDRESS=0.0.0.0 MQTT_BIND_PORT=1883 MQTT_TIMEOUT=10000 + +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus +WEB_APPLICATION_ENABLE=true +WEB_APPLICATION_TYPE=servlet +HTTP_BIND_PORT=8081 \ No newline at end of file diff --git a/docker/tb-node.env b/docker/tb-node.env index bc0a3f5ee5..e3393d41b1 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -10,3 +10,6 @@ REDIS_HOST=redis HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE=64 + +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus From ff14283c2059cc6d7b9622baef2d5225e7143f5a Mon Sep 17 00:00:00 2001 From: vzikratyi Date: Wed, 28 Apr 2021 15:13:53 +0300 Subject: [PATCH 25/32] Added default Grafana dashboards --- .../dashboards/attributes_cache.json | 330 ++++++ .../dashboards/core_and_js_metrics.json | 335 ++++++ .../provisioning/dashboards/db_metrics.json | 406 ++++++++ .../dashboards/rule_engine_latency.json | 386 +++++++ .../dashboards/rule_engine_metrics.json | 338 ++++++ .../dashboards/single_service_metrics.json | 982 ++++++++++++++++++ .../dashboards/transport_metrics.json | 532 ++++++++++ 7 files changed, 3309 insertions(+) create mode 100644 docker/monitoring/grafana/provisioning/dashboards/attributes_cache.json create mode 100644 docker/monitoring/grafana/provisioning/dashboards/core_and_js_metrics.json create mode 100644 docker/monitoring/grafana/provisioning/dashboards/db_metrics.json create mode 100644 docker/monitoring/grafana/provisioning/dashboards/rule_engine_latency.json create mode 100644 docker/monitoring/grafana/provisioning/dashboards/rule_engine_metrics.json create mode 100644 docker/monitoring/grafana/provisioning/dashboards/single_service_metrics.json create mode 100644 docker/monitoring/grafana/provisioning/dashboards/transport_metrics.json diff --git a/docker/monitoring/grafana/provisioning/dashboards/attributes_cache.json b/docker/monitoring/grafana/provisioning/dashboards/attributes_cache.json new file mode 100644 index 0000000000..2e21b9cb6e --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/attributes_cache.json @@ -0,0 +1,330 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "increase(attributes_cache_total[1m])", + "interval": "", + "legendFormat": "{{job}} {{result}}", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum (increase(attributes_cache_total{result=\"hit\"}[1m]))", + "interval": "", + "legendFormat": "Hits", + "queryType": "randomWalk", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (increase(attributes_cache_total{result=\"miss\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "Misses", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Summarized Cache Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "100*\n(\n sum(increase(attributes_cache_total{result=\"hit\"}[1m])) / \n ( \n sum(increase(attributes_cache_total{result=\"hit\"}[1m]))\n + sum(increase(attributes_cache_total{result=\"miss\"}[1m]))\n )\n)", + "interval": "", + "legendFormat": "Hit Ratio, %", + "queryType": "randomWalk", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Attributes Cache Hit Ratio %", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Attributes Cache", + "uid": "dxj2OYTMk", + "version": 2 +} \ No newline at end of file diff --git a/docker/monitoring/grafana/provisioning/dashboards/core_and_js_metrics.json b/docker/monitoring/grafana/provisioning/dashboards/core_and_js_metrics.json new file mode 100644 index 0000000000..e04a5c2d84 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/core_and_js_metrics.json @@ -0,0 +1,335 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(core_producer_total[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Core Producer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(core_total[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Core Starts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(jsInvoke_total[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JsInvoke Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Core and JS Metrics", + "uid": "lewbrlwjerwkj2", + "version": 2 +} \ No newline at end of file diff --git a/docker/monitoring/grafana/provisioning/dashboards/db_metrics.json b/docker/monitoring/grafana/provisioning/dashboards/db_metrics.json new file mode 100644 index 0000000000..f5983cf304 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/db_metrics.json @@ -0,0 +1,406 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 7, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(attributes_queue_0_total[1m]))", + "interval": "", + "legendFormat": "queue-0 {{statsName}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(attributes_queue_1_total[1m]))", + "interval": "", + "legendFormat": "queue-1 {{statsName}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(attributes_queue_2_total[1m]))", + "interval": "", + "legendFormat": "queue-2 {{statsName}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (rate(attributes_queue_3_total[1m]))", + "interval": "", + "legendFormat": "queue-3 {{statsName}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Attributes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_queue_0_total[1m]))", + "interval": "", + "legendFormat": "queue-0 {{statsName}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_queue_1_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-1 {{statsName}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_queue_2_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-2 {{statsName}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_queue_3_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-3 {{statsName}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 11 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_0_total[1m]))", + "interval": "", + "legendFormat": "queue-0 {{statsName}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_1_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-1 {{statsName}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_2_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-2 {{statsName}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_3_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-3 {{statsName}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TS Latest", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "DB Metrics", + "uid": "lewbrlsssswjerwkj", + "version": 2 +} \ No newline at end of file diff --git a/docker/monitoring/grafana/provisioning/dashboards/rule_engine_latency.json b/docker/monitoring/grafana/provisioning/dashboards/rule_engine_latency.json new file mode 100644 index 0000000000..1821eeba41 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/rule_engine_latency.json @@ -0,0 +1,386 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (job,quantile) (ruleEngine_Main_seconds)*1000", + "interval": "", + "legendFormat": "Main {{job}} Quantile - {{quantile}}, ms", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by (job,quantile) (ruleEngine_HighPriority_seconds)*1000", + "hide": false, + "interval": "", + "legendFormat": "HighPriority {{job}} Quantile - {{quantile}}, ms", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by (job,quantile) (ruleEngine_SequentialByOriginator_seconds)*1000", + "hide": false, + "interval": "", + "legendFormat": "SequentialByOriginator {{job}} Quantile - {{quantile}}, ms", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Quantiles Latency, ms", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "ruleEngine_Main_seconds_max *1000", + "interval": "", + "legendFormat": "Max - {{job}}, ms", + "refId": "A" + }, + { + "exemplar": true, + "expr": "ruleEngine_HighPriority_seconds_max *1000", + "hide": false, + "interval": "", + "legendFormat": "HighPriority - {{job}}, ms", + "refId": "B" + }, + { + "exemplar": true, + "expr": "ruleEngine_SequentialByOriginator_seconds_max *1000", + "hide": false, + "interval": "", + "legendFormat": "SequentialByOriginator - {{job}}, ms", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Max Latency, ms", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "(increase(ruleEngine_Main_seconds_sum[1m]) / increase(ruleEngine_Main_seconds_count[1m])) * 1000", + "interval": "", + "legendFormat": "Main {{job}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "(increase(ruleEngine_HighPriority_seconds_sum[1m]) / increase(ruleEngine_HighPriority_seconds_count[1m])) * 1000", + "hide": false, + "interval": "", + "legendFormat": "HighPriority {{job}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "(increase(ruleEngine_SequentialByOriginator_seconds_sum[1m]) / increase(ruleEngine_SequentialByOriginator_seconds_count[1m])) * 1000", + "hide": false, + "interval": "", + "legendFormat": "SequentialByOriginator {{job}}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Average by 1m", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "10s", + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rule Engine Latency", + "uid": "-qNMB1SGz", + "version": 2 +} \ No newline at end of file diff --git a/docker/monitoring/grafana/provisioning/dashboards/rule_engine_metrics.json b/docker/monitoring/grafana/provisioning/dashboards/rule_engine_metrics.json new file mode 100644 index 0000000000..9fc1ba2678 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/rule_engine_metrics.json @@ -0,0 +1,338 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName)(increase(ruleEngine_Main_total[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[Main] Rule Engine Queue Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName)(increase(ruleEngine_HighPriority_total[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[HighPriority] Rule Engine Queue Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName)(increase(ruleEngine_SequentialByOriginator_total[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[SequentialByOriginator] Rule Engine Queue Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Rule Engine Metrics", + "uid": "lewbrlwjerwkj1", + "version": 2 +} \ No newline at end of file diff --git a/docker/monitoring/grafana/provisioning/dashboards/single_service_metrics.json b/docker/monitoring/grafana/provisioning/dashboards/single_service_metrics.json new file mode 100644 index 0000000000..df33e3efb5 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/single_service_metrics.json @@ -0,0 +1,982 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 6, + "iteration": 1619603301193, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(core_producer_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Core Producer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(transport_producer_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transport Producer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ruleEngine_producer_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rule Engine Producer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(core_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Core Starts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(transport_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transport Consumer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName)(increase(ruleEngine_Main_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[Main] Rule Engine Queue Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName)(increase(ruleEngine_HighPriority_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[HighPriority] Rule Engine Queue Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName)(increase(ruleEngine_SequentialByOriginator_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[SequentialByOriginator] Rule Engine Queue Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 32 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(jsInvoke_total{job=\"$job\"}[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JsInvoke Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "selected": true, + "text": "tb-mqtt-transport1", + "value": "tb-mqtt-transport1" + }, + "description": null, + "error": null, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "job", + "options": [ + { + "selected": false, + "text": "tb-core1", + "value": "tb-core1" + }, + { + "selected": false, + "text": "tb-core2", + "value": "tb-core2" + }, + { + "selected": false, + "text": "tb-rule-engine1", + "value": "tb-rule-engine1" + }, + { + "selected": false, + "text": "tb-rule-engine2", + "value": "tb-rule-engine2" + }, + { + "selected": true, + "text": "tb-mqtt-transport1", + "value": "tb-mqtt-transport1" + }, + { + "selected": false, + "text": "tb-mqtt-transport2", + "value": "tb-mqtt-transport2" + }, + { + "selected": false, + "text": "tb-http-transport1", + "value": "tb-http-transport1" + }, + { + "selected": false, + "text": "tb-http-transport2", + "value": "tb-http-transport2" + }, + { + "selected": false, + "text": "tb-coap-transport", + "value": "tb-coap-transport" + } + ], + "query": "tb-core1,tb-core2,tb-rule-engine1,tb-rule-engine2,tb-mqtt-transport1,tb-mqtt-transport2,tb-http-transport1,tb-http-transport2,tb-coap-transport", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Single Service Metrics", + "uid": "lewbrddddlwjerwkj", + "version": 1 +} \ No newline at end of file diff --git a/docker/monitoring/grafana/provisioning/dashboards/transport_metrics.json b/docker/monitoring/grafana/provisioning/dashboards/transport_metrics.json new file mode 100644 index 0000000000..3027912a59 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/transport_metrics.json @@ -0,0 +1,532 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum (increase(transport_producer_total{statsName=\"totalMsgs\"}[1m]))", + "interval": "", + "legendFormat": "producer", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum (increase(transport_total{statsName=\"totalMsgs\"}[1m]))", + "hide": false, + "interval": "", + "legendFormat": "consumer", + "refId": "B" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Transport Producer/Consumer Comparison", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "(increase(transport_total[1m]))", + "interval": "", + "legendFormat": "{{job}} - {{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transport Consumer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "(increase(transport_producer_total[1m]))", + "interval": "", + "legendFormat": "{{job}} - {{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Transport Producer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "(increase(core_producer_total[1m]))", + "interval": "", + "legendFormat": "{{job}} - {{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Core Producer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "hiddenSeries": false, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "(increase(ruleEngine_producer_total[1m]))", + "interval": "", + "legendFormat": "{{job}} - {{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rule Engine Producer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Transport Metrics", + "uid": "lewbrlwjerwkj", + "version": 4 +} \ No newline at end of file From e618f2382aec55e272fb5f1572876fd77f078f93 Mon Sep 17 00:00:00 2001 From: vzikratyi Date: Wed, 28 Apr 2021 18:28:11 +0300 Subject: [PATCH 26/32] Added dashboard for 'hybrid' DB setup --- .../dashboards/hybrid_db_metrics.json | 474 ++++++++++++++++++ 1 file changed, 474 insertions(+) create mode 100644 docker/monitoring/grafana/provisioning/dashboards/hybrid_db_metrics.json diff --git a/docker/monitoring/grafana/provisioning/dashboards/hybrid_db_metrics.json b/docker/monitoring/grafana/provisioning/dashboards/hybrid_db_metrics.json new file mode 100644 index 0000000000..ed6bd41c00 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/hybrid_db_metrics.json @@ -0,0 +1,474 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 8, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(attributes_queue_0_total[1m]))", + "interval": "", + "legendFormat": "queue-0 {{statsName}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(attributes_queue_1_total[1m]))", + "interval": "", + "legendFormat": "queue-1 {{statsName}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(attributes_queue_2_total[1m]))", + "interval": "", + "legendFormat": "queue-2 {{statsName}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (rate(attributes_queue_3_total[1m]))", + "interval": "", + "legendFormat": "queue-3 {{statsName}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Attributes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum (rateExecutor_currBuffer)", + "interval": "", + "legendFormat": "current buffer", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Cassandra Current Buffer", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hiddenSeries": false, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by (statsName) (increase(rateExecutor_total[1m]))", + "interval": "", + "legendFormat": "{{statsName}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TS", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hiddenSeries": false, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.4", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_0_total[1m]))", + "interval": "", + "legendFormat": "queue-0 {{statsName}}", + "refId": "A" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_1_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-1 {{statsName}}", + "refId": "B" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_2_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-2 {{statsName}}", + "refId": "C" + }, + { + "exemplar": true, + "expr": "sum by(statsName) (increase(ts_latest_queue_3_total[1m]))", + "hide": false, + "interval": "", + "legendFormat": "queue-3 {{statsName}}", + "refId": "D" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "TS Latest", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 27, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Hybrid DB Metrics", + "uid": "lewbrlsssswje", + "version": 2 +} \ No newline at end of file From fbf2d3efdbd66a3cf5f1513439fe220c1c5cdeec Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 29 Apr 2021 14:55:27 +0300 Subject: [PATCH 27/32] Fix entities filtering by kv when search value is empty (#4515) * Fix entities filtering by kv when search value is empty * Refactor --- .../dao/sql/query/EntityKeyMapping.java | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java index a759d0de74..caed48758a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/EntityKeyMapping.java @@ -243,23 +243,21 @@ public class EntityKeyMapping { } else { entityTypeStr = "'" + entityType.name() + "'"; } - ctx.addStringParameter(alias + "_key_id", entityKey.getKey()); + ctx.addStringParameter(getKeyId(), entityKey.getKey()); String filterQuery = toQueries(ctx, entityFilter.getType()) .filter(StringUtils::isNotEmpty) .collect(Collectors.joining(" and ")); - if (StringUtils.isEmpty(filterQuery)) { - filterQuery = ""; - } else { + if (StringUtils.isNotEmpty(filterQuery)) { filterQuery = " AND (" + filterQuery + ")"; } if (entityKey.getType().equals(EntityKeyType.TIME_SERIES)) { - String join = hasFilter() ? "inner join" : "left join"; + String join = (hasFilter() && hasFilterValues(ctx)) ? "inner join" : "left join"; return String.format("%s ts_kv_latest %s ON %s.entity_id=entities.id AND %s.key = (select key_id from ts_kv_dictionary where key = :%s_key_id) %s", join, alias, alias, alias, alias, filterQuery); } else { String query; if (!entityKey.getType().equals(EntityKeyType.ATTRIBUTE)) { - String join = hasFilter() ? "inner join" : "left join"; + String join = (hasFilter() && hasFilterValues(ctx)) ? "inner join" : "left join"; query = String.format("%s attribute_kv %s ON %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key=:%s_key_id ", join, alias, alias, alias, entityTypeStr, alias, alias); String scope; @@ -272,7 +270,7 @@ public class EntityKeyMapping { } query = String.format("%s AND %s.attribute_type='%s' %s", query, alias, scope, filterQuery); } else { - String join = hasFilter() ? "join LATERAL" : "left join LATERAL"; + String join = (hasFilter() && hasFilterValues(ctx)) ? "join LATERAL" : "left join LATERAL"; query = String.format("%s (select * from attribute_kv %s WHERE %s.entity_id=entities.id AND %s.entity_type=%s AND %s.attribute_key=:%s_key_id %s " + "ORDER BY %s.last_update_ts DESC limit 1) as %s ON true", join, alias, alias, alias, entityTypeStr, alias, alias, filterQuery, alias, alias); @@ -281,15 +279,26 @@ public class EntityKeyMapping { } } + private boolean hasFilterValues(QueryContext ctx) { + return Arrays.stream(ctx.getParameterNames()).anyMatch(parameterName -> { + return !parameterName.equals(getKeyId()) && parameterName.startsWith(alias); + }); + } + + private String getKeyId() { + return alias + "_key_id"; + } + public static String buildSelections(List mappings, EntityFilterType filterType, EntityType entityType) { return mappings.stream().map(mapping -> mapping.toSelection(filterType, entityType)).collect( Collectors.joining(", ")); } public static String buildLatestJoins(QueryContext ctx, EntityFilter entityFilter, EntityType entityType, List latestMappings, boolean countQuery) { - return latestMappings.stream().filter(mapping -> !countQuery || mapping.hasFilter()) - .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)).collect( - Collectors.joining(" ")); + return latestMappings.stream() + .filter(mapping -> !countQuery || mapping.hasFilter()) + .map(mapping -> mapping.toLatestJoin(ctx, entityFilter, entityType)) + .collect(Collectors.joining(" ")); } public static String buildQuery(QueryContext ctx, List mappings, EntityFilterType filterType) { From 52c34068aef87f1c5a9bdb35a93daa1cd95d5743 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 29 Apr 2021 16:55:05 +0300 Subject: [PATCH 28/32] UI: Move Resource library to system settings menu --- ui-ngx/src/app/core/services/menu.service.ts | 94 ++++++++++--------- .../home/pages/admin/admin-routing.module.ts | 26 +++-- .../modules/home/pages/admin/admin.module.ts | 4 +- .../resources-library-table-config.resolve.ts | 2 +- .../resource/resources-library.component.html | 0 .../resource/resources-library.component.ts | 0 .../modules/home/pages/home-pages.module.ts | 2 - .../pages/resource/resource-routing.module.ts | 48 ---------- .../home/pages/resource/resource.module.ts | 33 ------- .../assets/locale/locale.constant-en_US.json | 1 - 10 files changed, 75 insertions(+), 135 deletions(-) rename ui-ngx/src/app/modules/home/pages/{ => admin}/resource/resources-library-table-config.resolve.ts (98%) rename ui-ngx/src/app/modules/home/pages/{ => admin}/resource/resources-library.component.html (100%) rename ui-ngx/src/app/modules/home/pages/{ => admin}/resource/resources-library.component.ts (100%) delete mode 100644 ui-ngx/src/app/modules/home/pages/resource/resource-routing.module.ts delete mode 100644 ui-ngx/src/app/modules/home/pages/resource/resource.module.ts diff --git a/ui-ngx/src/app/core/services/menu.service.ts b/ui-ngx/src/app/core/services/menu.service.ts index f1f286ca16..07a6d8f237 100644 --- a/ui-ngx/src/app/core/services/menu.service.ts +++ b/ui-ngx/src/app/core/services/menu.service.ts @@ -103,19 +103,12 @@ export class MenuService { path: '/widgets-bundles', icon: 'now_widgets' }, - { - id: guid(), - name: 'resource.resources-library', - type: 'link', - path: '/resources-library', - icon: 'folder' - }, { id: guid(), name: 'admin.system-settings', type: 'toggle', path: '/settings', - height: '200px', + height: '240px', icon: 'settings', pages: [ { @@ -152,6 +145,13 @@ export class MenuService { type: 'link', path: '/settings/oauth2', icon: 'security' + }, + { + id: guid(), + name: 'resource.resources-library', + type: 'link', + path: '/settings/resources-library', + icon: 'folder' } ] } @@ -188,16 +188,6 @@ export class MenuService { } ] }, - { - name: 'resource.management', - places: [ - { - name: 'resource.resources-library', - icon: 'folder', - path: '/resources-library' - } - ] - }, { name: 'admin.system-settings', places: [ @@ -225,6 +215,11 @@ export class MenuService { name: 'admin.oauth2.oauth2', icon: 'security', path: '/settings/oauth2' + }, + { + name: 'resource.resources-library', + icon: 'folder', + path: '/resources-library' } ] } @@ -337,20 +332,6 @@ export class MenuService { path: '/dashboards', icon: 'dashboards' }, - { - id: guid(), - name: 'resource.resources-library', - type: 'link', - path: '/resources-library', - icon: 'folder' - }, - { - id: guid(), - name: 'admin.home-settings', - type: 'link', - path: '/settings/home', - icon: 'settings_applications' - }, { id: guid(), name: 'audit-log.audit-logs', @@ -365,6 +346,30 @@ export class MenuService { path: '/usage', icon: 'insert_chart', notExact: true + }, + { + id: guid(), + name: 'admin.system-settings', + type: 'toggle', + path: '/settings', + height: '80px', + icon: 'settings', + pages: [ + { + id: guid(), + name: 'admin.home-settings', + type: 'link', + path: '/settings/home', + icon: 'settings_applications' + }, + { + id: guid(), + name: 'resource.resources-library', + type: 'link', + path: '/settings/resources-library', + icon: 'folder' + } + ] } ); return sections; @@ -455,16 +460,6 @@ export class MenuService { ); } homeSections.push( - { - name: 'resource.management', - places: [ - { - name: 'resource.resources-library', - icon: 'folder', - path: '/resources-library' - } - ] - }, { name: 'dashboard.management', places: [ @@ -494,6 +489,21 @@ export class MenuService { path: '/usage' } ] + }, + { + name: 'admin.system-settings', + places: [ + { + name: 'admin.home-settings', + icon: 'settings_applications', + path: '/settings/home' + }, + { + name: 'resource.resources-library', + icon: 'folder', + path: '/settings/resources-library' + } + ] } ); return homeSections; diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts index 1928a1f404..8de41af9c2 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin-routing.module.ts @@ -23,16 +23,12 @@ import { Authority } from '@shared/models/authority.enum'; import { GeneralSettingsComponent } from '@modules/home/pages/admin/general-settings.component'; import { SecuritySettingsComponent } from '@modules/home/pages/admin/security-settings.component'; import { OAuth2SettingsComponent } from '@home/pages/admin/oauth2-settings.component'; -import { User } from '@shared/models/user.model'; -import { Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { UserService } from '@core/http/user.service'; import { Observable } from 'rxjs'; -import { getCurrentAuthUser } from '@core/auth/auth.selectors'; import { OAuth2Service } from '@core/http/oauth2.service'; -import { UserProfileResolver } from '@home/pages/profile/profile-routing.module'; import { SmsProviderComponent } from '@home/pages/admin/sms-provider.component'; import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; +import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; +import { ResourcesLibraryTableConfigResolver } from '@home/pages/admin/resource/resources-library-table-config.resolve'; @Injectable() export class OAuth2LoginProcessingUrlResolver implements Resolve { @@ -146,6 +142,21 @@ const routes: Routes = [ icon: 'settings_applications' } } + }, + { + path: 'resources-library', + component: EntitiesTableComponent, + data: { + auth: [Authority.TENANT_ADMIN, Authority.SYS_ADMIN], + title: 'resource.resources-library', + breadcrumb: { + label: 'resource.resources-library', + icon: 'folder' + } + }, + resolve: { + entitiesTableConfig: ResourcesLibraryTableConfigResolver + } } ] } @@ -155,7 +166,8 @@ const routes: Routes = [ imports: [RouterModule.forChild(routes)], exports: [RouterModule], providers: [ - OAuth2LoginProcessingUrlResolver + OAuth2LoginProcessingUrlResolver, + ResourcesLibraryTableConfigResolver ] }) export class AdminRoutingModule { } diff --git a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts index 1671220cfe..d97471a2a0 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/admin.module.ts @@ -27,6 +27,7 @@ import { OAuth2SettingsComponent } from '@modules/home/pages/admin/oauth2-settin import { SmsProviderComponent } from '@home/pages/admin/sms-provider.component'; import { SendTestSmsDialogComponent } from '@home/pages/admin/send-test-sms-dialog.component'; import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component'; +import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; @NgModule({ declarations: @@ -37,7 +38,8 @@ import { HomeSettingsComponent } from '@home/pages/admin/home-settings.component SendTestSmsDialogComponent, SecuritySettingsComponent, OAuth2SettingsComponent, - HomeSettingsComponent + HomeSettingsComponent, + ResourcesLibraryComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/resource/resources-library-table-config.resolve.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts similarity index 98% rename from ui-ngx/src/app/modules/home/pages/resource/resources-library-table-config.resolve.ts rename to ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts index d6759310ac..089d4b3d82 100644 --- a/ui-ngx/src/app/modules/home/pages/resource/resources-library-table-config.resolve.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library-table-config.resolve.ts @@ -32,7 +32,7 @@ import { getCurrentAuthUser } from '@core/auth/auth.selectors'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { Authority } from '@shared/models/authority.enum'; -import { ResourcesLibraryComponent } from '@home/pages/resource/resources-library.component'; +import { ResourcesLibraryComponent } from '@home/pages/admin/resource/resources-library.component'; import { PageLink } from '@shared/models/page/page-link'; import { EntityAction } from '@home/models/entity/entity-component.models'; import { map } from 'rxjs/operators'; diff --git a/ui-ngx/src/app/modules/home/pages/resource/resources-library.component.html b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html similarity index 100% rename from ui-ngx/src/app/modules/home/pages/resource/resources-library.component.html rename to ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.html diff --git a/ui-ngx/src/app/modules/home/pages/resource/resources-library.component.ts b/ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts similarity index 100% rename from ui-ngx/src/app/modules/home/pages/resource/resources-library.component.ts rename to ui-ngx/src/app/modules/home/pages/admin/resource/resources-library.component.ts diff --git a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts index 774eccf62f..050b71db83 100644 --- a/ui-ngx/src/app/modules/home/pages/home-pages.module.ts +++ b/ui-ngx/src/app/modules/home/pages/home-pages.module.ts @@ -35,7 +35,6 @@ import { modulesMap } from '../../common/modules-map'; import { DeviceProfileModule } from './device-profile/device-profile.module'; import { ApiUsageModule } from '@home/pages/api-usage/api-usage.module'; import { EdgeModule } from '@home/pages/edge/edge.module'; -import { ResourceModule } from '@home/pages/resource/resource.module'; import { FirmwareModule } from '@home/pages/firmware/firmware.module'; @NgModule({ @@ -56,7 +55,6 @@ import { FirmwareModule } from '@home/pages/firmware/firmware.module'; DashboardModule, AuditLogModule, ApiUsageModule, - ResourceModule, FirmwareModule, UserModule ], diff --git a/ui-ngx/src/app/modules/home/pages/resource/resource-routing.module.ts b/ui-ngx/src/app/modules/home/pages/resource/resource-routing.module.ts deleted file mode 100644 index 4eab81e2f2..0000000000 --- a/ui-ngx/src/app/modules/home/pages/resource/resource-routing.module.ts +++ /dev/null @@ -1,48 +0,0 @@ -/// -/// Copyright © 2016-2021 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { RouterModule, Routes } from '@angular/router'; -import { EntitiesTableComponent } from '@home/components/entity/entities-table.component'; -import { Authority } from '@shared/models/authority.enum'; -import { NgModule } from '@angular/core'; -import { ResourcesLibraryTableConfigResolver } from './resources-library-table-config.resolve'; - -const routes: Routes = [ - { - path: 'resources-library', - component: EntitiesTableComponent, - data: { - auth: [Authority.TENANT_ADMIN, Authority.SYS_ADMIN], - title: 'resource.resources-library', - breadcrumb: { - label: 'resource.resources-library', - icon: 'folder' - } - }, - resolve: { - entitiesTableConfig: ResourcesLibraryTableConfigResolver - } - } -]; - -@NgModule({ - imports: [RouterModule.forChild(routes)], - exports: [RouterModule], - providers: [ - ResourcesLibraryTableConfigResolver - ] -}) -export class ResourcesLibraryRoutingModule{ } diff --git a/ui-ngx/src/app/modules/home/pages/resource/resource.module.ts b/ui-ngx/src/app/modules/home/pages/resource/resource.module.ts deleted file mode 100644 index 4cf31af12c..0000000000 --- a/ui-ngx/src/app/modules/home/pages/resource/resource.module.ts +++ /dev/null @@ -1,33 +0,0 @@ -/// -/// Copyright © 2016-2021 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { NgModule } from '@angular/core'; -import { CommonModule } from '@angular/common'; -import { ResourcesLibraryRoutingModule } from '@home/pages/resource/resource-routing.module'; -import { SharedModule } from '@shared/shared.module'; -import { HomeComponentsModule } from '@home/components/home-components.module'; -import { ResourcesLibraryComponent } from './resources-library.component'; - -@NgModule({ - declarations: [ResourcesLibraryComponent], - imports: [ - CommonModule, - SharedModule, - HomeComponentsModule, - ResourcesLibraryRoutingModule - ] -}) -export class ResourceModule { } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 08335a0175..313919c4f6 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2238,7 +2238,6 @@ "drop-file": "Drop a resource file or click to select a file to upload.", "empty": "Resource is empty", "export": "Export resource", - "management": "Resource management", "no-resource-matching": "No resource matching '{{widgetsBundle}}' were found.", "no-resource-text": "No resources found", "open-widgets-bundle": "Open widgets bundle", From 99a25cab00a38bcff520f25ca1c2c1cda0f25cb2 Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Thu, 29 Apr 2021 18:25:23 +0300 Subject: [PATCH 29/32] UI: Fixed field name in LC_EVENT; Added error field reset at hiding in event filters (#4453) * UI: Fixed field name in LC_EVENT; Added error field reset at hiding in event filters * UI: Updated event filter interface * UI: Add reset page in event filter --- .../components/event/event-filter-panel.component.html | 3 ++- .../components/event/event-filter-panel.component.ts | 6 ++++++ .../home/components/event/event-table-config.ts | 3 ++- ui-ngx/src/app/shared/models/event.models.ts | 10 ++++++---- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.html b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.html index 6a017f214e..579301d086 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.html @@ -39,7 +39,8 @@ - + {{ 'event.has-error' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts index f60f7a09f7..954253b459 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-filter-panel.component.ts @@ -105,5 +105,11 @@ export class EventFilterPanelComponent { cancel() { this.overlayRef.dispose(); } + + changeIsError(value: boolean | string) { + if (this.conditionError && value === '') { + this.eventFilterFormGroup.get('error').reset('', {emitEvent: false}); + } + } } diff --git a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts index 1dc663684f..8caa6c7aab 100644 --- a/ui-ngx/src/app/modules/home/components/event/event-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/event/event-table-config.ts @@ -288,7 +288,7 @@ export class EventTableConfig extends EntityTableConfig { break; case EventType.LC_EVENT: this.filterColumns.push( - {key: 'method', title: 'event.event'}, + {key: 'event', title: 'event.event'}, {key: 'status', title: 'event.status'}, {key: 'error', title: 'event.error'} ); @@ -356,6 +356,7 @@ export class EventTableConfig extends EntityTableConfig { componentRef.onDestroy(() => { if (componentRef.instance.result && !isEqual(this.filterParams, componentRef.instance.result.filterParams)) { this.filterParams = componentRef.instance.result.filterParams; + this.table.paginator.pageIndex = 0; this.table.updateData(); } }); diff --git a/ui-ngx/src/app/shared/models/event.models.ts b/ui-ngx/src/app/shared/models/event.models.ts index b801bab750..ac7d87c0c4 100644 --- a/ui-ngx/src/app/shared/models/event.models.ts +++ b/ui-ngx/src/app/shared/models/event.models.ts @@ -91,17 +91,18 @@ export interface BaseFilterEventBody { export interface ErrorFilterEventBody extends BaseFilterEventBody { method?: string; + error?: string; } export interface LcFilterEventEventBody extends BaseFilterEventBody { - method?: string; + event?: string; status?: string; - isError?: boolean; + error?: string; } export interface StatsFilterEventBody extends BaseFilterEventBody { - messagesProcessed?: string; - errorsOccurred?: string; + messagesProcessed?: number; + errorsOccurred?: number; } export interface DebugFilterRuleNodeEventBody extends BaseFilterEventBody { @@ -114,6 +115,7 @@ export interface DebugFilterRuleNodeEventBody extends BaseFilterEventBody { dataSearch?: string; metadataSearch?: string; isError?: boolean; + error?: string; } export type FilterEventBody = ErrorFilterEventBody & LcFilterEventEventBody & StatsFilterEventBody & DebugFilterRuleNodeEventBody; From 9dcb7b9b5bc13c39b7531322259ee90ceb52050c Mon Sep 17 00:00:00 2001 From: Vladyslav Date: Thu, 29 Apr 2021 18:25:38 +0300 Subject: [PATCH 30/32] UI: Refactoring for the build rule node ui (#4460) * UI: Refactoring LWM2M; fixed build ui rule node * UI: Refactoring LWM2M security config --- .../device/device-credentials.component.html | 4 +- .../device/device-credentials.component.ts | 75 ++-- ...ecurity-config-lwm2m-server.component.html | 79 ++++ ...ecurity-config-lwm2m-server.component.scss | 20 + .../security-config-lwm2m-server.component.ts | 176 ++++++++ .../security-config-lwm2m.component.html | 143 +++++++ .../security-config-lwm2m.component.scss | 34 ++ .../device/security-config-lwm2m.component.ts | 257 ++++++++++++ .../home/components/home-components.module.ts | 6 + .../lwm2m-attributes-dialog.component.ts | 27 +- .../lwm2m-attributes-key-list.component.ts | 30 +- .../lwm2m/lwm2m-attributes.component.ts | 41 +- .../lwm2m-device-config-server.component.ts | 18 +- ...ofile-transport-configuration.component.ts | 31 +- ...m-object-add-instances-dialog.component.ts | 14 +- ...m2m-object-add-instances-list.component.ts | 11 +- .../lwm2m/lwm2m-object-list.component.ts | 47 +-- ...serve-attr-telemetry-resource.component.ts | 19 +- .../lwm2m-observe-attr-telemetry.component.ts | 24 +- .../lwm2m/lwm2m-profile-components.module.ts | 26 +- .../lwm2m/lwm2m-profile-config.models.ts | 4 +- .../home/pages/device/device.module.ts | 6 +- .../security-config-server.component.html | 101 ----- .../lwm2m/security-config-server.component.ts | 142 ------- .../lwm2m/security-config.component.html | 180 --------- .../device/lwm2m/security-config.component.ts | 382 ------------------ .../models/lwm2m-security-config.models.ts} | 66 ++- .../assets/locale/locale.constant-en_US.json | 15 +- 28 files changed, 925 insertions(+), 1053 deletions(-) create mode 100644 ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.html create mode 100644 ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.ts create mode 100644 ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.html create mode 100644 ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.scss create mode 100644 ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.ts delete mode 100644 ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.html delete mode 100644 ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.ts delete mode 100644 ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.html delete mode 100644 ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.ts rename ui-ngx/src/app/{modules/home/pages/device/lwm2m/security-config.models.ts => shared/models/lwm2m-security-config.models.ts} (60%) diff --git a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html index faa2dbbce4..db28ff0453 100644 --- a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html +++ b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.html @@ -20,7 +20,7 @@ device.credentials-type - {{ credentialTypeNamesMap.get(deviceCredentialsType[credentialsType]) }} + {{ credentialTypeNamesMap.get(credentialsType) }} @@ -85,7 +85,7 @@ device.lwm2m-value diff --git a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts index edc331d338..17ad828b48 100644 --- a/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts +++ b/ui-ngx/src/app/modules/home/components/device/device-credentials.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {Component, forwardRef, Input, OnDestroy, OnInit} from '@angular/core'; +import { Component, forwardRef, Input, OnDestroy, OnInit } from '@angular/core'; import { ControlValueAccessor, FormBuilder, @@ -33,9 +33,9 @@ import { DeviceCredentials, DeviceCredentialsType } from '@shared/models/device.models'; -import {Subscription} from 'rxjs'; -import {distinctUntilChanged} from 'rxjs/operators'; -import {SecurityConfigComponent} from '@home/pages/device/lwm2m/security-config.component'; +import { Subject } from 'rxjs'; +import { distinctUntilChanged, takeUntil } from 'rxjs/operators'; +import { SecurityConfigLwm2mComponent } from '@home/components/device/security-config-lwm2m.component'; import { ClientSecurityConfig, DEFAULT_END_POINT, @@ -44,10 +44,10 @@ import { getDefaultSecurityConfig, JSON_ALL_CONFIG, validateSecurityConfig -} from '@home/pages/device/lwm2m/security-config.models'; -import {TranslateService} from '@ngx-translate/core'; -import {MatDialog} from '@angular/material/dialog'; -import {isDefinedAndNotNull} from '@core/utils'; +} from '@shared/models/lwm2m-security-config.models'; +import { TranslateService } from '@ngx-translate/core'; +import { MatDialog } from '@angular/material/dialog'; +import { isDefinedAndNotNull } from '@core/utils'; @Component({ selector: 'tb-device-credentials', @@ -67,20 +67,16 @@ import {isDefinedAndNotNull} from '@core/utils'; }) export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, Validator, OnDestroy { - deviceCredentialsFormGroup: FormGroup; - - subscriptions: Subscription[] = []; - @Input() disabled: boolean; - deviceCredentials: DeviceCredentials = null; + private destroy$ = new Subject(); - submitted = false; + deviceCredentialsFormGroup: FormGroup; deviceCredentialsType = DeviceCredentialsType; - credentialsTypes = Object.keys(DeviceCredentialsType); + credentialsTypes = Object.values(DeviceCredentialsType); credentialTypeNamesMap = credentialTypeNames; @@ -102,16 +98,17 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, }, {validators: this.atLeastOne(Validators.required, ['clientId', 'userName'])}) }); this.deviceCredentialsFormGroup.get('credentialsBasic').disable(); - this.subscriptions.push( - this.deviceCredentialsFormGroup.valueChanges.pipe(distinctUntilChanged()).subscribe(() => { - this.updateView(); - }) - ); - this.subscriptions.push( - this.deviceCredentialsFormGroup.get('credentialsType').valueChanges.subscribe(() => { - this.credentialsTypeChanged(); - }) - ); + this.deviceCredentialsFormGroup.valueChanges.pipe( + distinctUntilChanged(), + takeUntil(this.destroy$) + ).subscribe(() => { + this.updateView(); + }); + this.deviceCredentialsFormGroup.get('credentialsType').valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe((type) => { + this.credentialsTypeChanged(type); + }); } ngOnInit(): void { @@ -121,12 +118,12 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, } ngOnDestroy() { - this.subscriptions.forEach(s => s.unsubscribe()); + this.destroy$.next(); + this.destroy$.complete(); } writeValue(value: DeviceCredentials | null): void { if (isDefinedAndNotNull(value)) { - this.deviceCredentials = value; let credentialsBasic = {clientId: null, userName: null, password: null}; let credentialsValue = null; if (value.credentialsType === DeviceCredentialsType.MQTT_BASIC) { @@ -179,10 +176,11 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, }; } - credentialsTypeChanged(): void { + credentialsTypeChanged(credentialsType: DeviceCredentialsType): void { + const credentialsValue = credentialsType === DeviceCredentialsType.LWM2M_CREDENTIALS ? this.lwm2mDefaultConfig : null; this.deviceCredentialsFormGroup.patchValue({ credentialsId: null, - credentialsValue: JSON.stringify(getDefaultSecurityConfig(), null, 2), + credentialsValue, credentialsBasic: {clientId: '', userName: '', password: ''} }); this.updateValidators(); @@ -264,7 +262,7 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, } } const credentialsId = this.deviceCredentialsFormGroup.get('credentialsId').value || DEFAULT_END_POINT; - this.dialog.open(SecurityConfigComponent, { + this.dialog.open(SecurityConfigLwm2mComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { @@ -275,8 +273,8 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, (res) => { if (res) { this.deviceCredentialsFormGroup.patchValue({ - credentialsValue: this.isDefautLw2mResponse(res[JSON_ALL_CONFIG]) ? null : JSON.stringify(res[JSON_ALL_CONFIG]), - credentialsId: this.isDefautLw2mResponse(res[END_POINT]) ? null : JSON.stringify(res[END_POINT]).split('\"').join('') + credentialsValue: this.isDefaultLw2mResponse(res[JSON_ALL_CONFIG]) ? null : JSON.stringify(res[JSON_ALL_CONFIG]), + credentialsId: this.isDefaultLw2mResponse(res[END_POINT]) ? null : JSON.stringify(res[END_POINT]).split('\"').join('') }); this.deviceCredentialsFormGroup.get('credentialsValue').markAsDirty(); } @@ -284,16 +282,19 @@ export class DeviceCredentialsComponent implements ControlValueAccessor, OnInit, ); } - private isDefautLw2mResponse(response: object): boolean { + private isDefaultLw2mResponse(response: object): boolean { return Object.keys(response).length === 0 || JSON.stringify(response) === '[{}]'; } private lwm2mConfigJsonValidator(control: FormControl) { - return validateSecurityConfig(control.value) ? null: {jsonError: {parsedJson: "error"}}; + return validateSecurityConfig(control.value) ? null : {jsonError: {parsedJson: 'error'}}; + } + + private get lwm2mDefaultConfig(): string { + return JSON.stringify(getDefaultSecurityConfig(), null, 2); } - lwm2mCredentialsValueTip (flag: boolean): string { - let jsonConfigDef = JSON.stringify(getDefaultSecurityConfig(), null, 2); - return !flag ? "" : 'Example (mode=\"NoSec\"):\n\r ' + jsonConfigDef; + lwm2mCredentialsValueTooltip(flag: boolean): string { + return !flag ? '' : 'Example (mode=\"NoSec\"):\n\r ' + this.lwm2mDefaultConfig; } } diff --git a/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.html b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.html new file mode 100644 index 0000000000..a40ce4cb69 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.html @@ -0,0 +1,79 @@ + +
+ + device.lwm2m-security-config.mode + + + {{ lwm2mSecurityTypeTranslationMap.get(securityType) }} + + + +
+ + {{ 'device.lwm2m-security-config.client-publicKey-or-id' | translate }} + + {{clientPublicKeyOrId.value?.length || 0}}/{{lenMaxClientPublicKeyOrId}} + + {{ 'device.lwm2m-security-config.client-publicKey-or-id-required' | translate }} + + + {{ 'device.lwm2m-security-config.client-publicKey-or-id-pattern' | translate }} + + + {{ 'device.lwm2m-security-config.client-publicKey-or-id-length' | translate: { + count: lenMaxClientPublicKeyOrId + } }} + + + + {{ 'device.lwm2m-security-config.client-secret-key' | translate }} + + {{clientSecretKey.value?.length || 0}}/{{lengthClientSecretKey}} + + {{ 'device.lwm2m-security-config.client-secret-key-required' | translate }} + + + {{ 'device.lwm2m-security-config.client-secret-key-pattern' | translate }} + + + {{ 'device.lwm2m-security-config.client-secret-key-length' | translate: { + count: lengthClientSecretKey + } }} + + +
+
diff --git a/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.scss b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.scss new file mode 100644 index 0000000000..1c55a86d8b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.scss @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host ::ng-deep { + textarea.mat-input-element.cdk-textarea-autosize { + box-sizing: content-box; + } +} diff --git a/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.ts b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.ts new file mode 100644 index 0000000000..b8e1b42ef4 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m-server.component.ts @@ -0,0 +1,176 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + +import { Component, forwardRef, OnDestroy } from '@angular/core'; +import { + ControlValueAccessor, + FormBuilder, + FormGroup, + NG_VALIDATORS, + NG_VALUE_ACCESSOR, + ValidationErrors, + Validator, + Validators +} from '@angular/forms'; +import { + KEY_REGEXP_HEX_DEC, + LEN_MAX_PRIVATE_KEY, + LEN_MAX_PSK, + LEN_MAX_PUBLIC_KEY_RPK, + LEN_MAX_PUBLIC_KEY_X509, + Lwm2mSecurityType, + Lwm2mSecurityTypeTranslationMap, + ServerSecurityConfig +} from '@shared/models/lwm2m-security-config.models'; +import { takeUntil } from 'rxjs/operators'; +import { Subject } from 'rxjs'; + +@Component({ + selector: 'tb-security-config-lwm2m-server', + templateUrl: './security-config-lwm2m-server.component.html', + styleUrls: ['./security-config-lwm2m-server.component.scss'], + providers: [ + { + provide: NG_VALUE_ACCESSOR, + useExisting: forwardRef(() => SecurityConfigLwm2mServerComponent), + multi: true + }, + { + provide: NG_VALIDATORS, + useExisting: forwardRef(() => SecurityConfigLwm2mServerComponent), + multi: true + } + ] +}) + +export class SecurityConfigLwm2mServerComponent implements OnDestroy, ControlValueAccessor, Validator { + + serverFormGroup: FormGroup; + securityConfigLwM2MType = Lwm2mSecurityType; + securityConfigLwM2MTypes = Object.values(Lwm2mSecurityType); + lwm2mSecurityTypeTranslationMap = Lwm2mSecurityTypeTranslationMap; + lenMinClientPublicKeyOrId = 0; + lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; + lengthClientSecretKey = LEN_MAX_PRIVATE_KEY; + + private destroy$ = new Subject(); + private propagateChange = (v: any) => {}; + + constructor(private fb: FormBuilder) { + this.serverFormGroup = this.fb.group({ + securityMode: [Lwm2mSecurityType.NO_SEC], + clientPublicKeyOrId: [''], + clientSecretKey: [''] + }); + this.serverFormGroup.get('securityMode').valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe((securityMode) => { + this.updateValidate(securityMode); + }); + + this.serverFormGroup.valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe((value) => { + this.propagateChange(value); + }); + } + + writeValue(value: any): void { + if (value) { + this.updateValueFields(value); + } + } + + registerOnChange(fn: any): void { + this.propagateChange = fn; + } + + registerOnTouched(fn: any): void { + } + + setDisabledState(isDisabled: boolean): void { + if (isDisabled) { + this.serverFormGroup.disable({emitEvent: false}); + } else { + this.serverFormGroup.enable({emitEvent: false}); + } + } + + validate(control): ValidationErrors | null { + return this.serverFormGroup.valid ? null : { + securityConfig: {valid: false} + }; + } + + ngOnDestroy() { + this.destroy$.next(); + this.destroy$.complete(); + } + + private updateValueFields(serverData: ServerSecurityConfig): void { + this.serverFormGroup.patchValue(serverData, {emitEvent: false}); + this.updateValidate(serverData.securityMode, true); + } + + private updateValidate(securityMode: Lwm2mSecurityType, initValue = false): void { + switch (securityMode) { + case Lwm2mSecurityType.NO_SEC: + this.serverFormGroup.get('clientPublicKeyOrId').clearValidators(); + this.serverFormGroup.get('clientSecretKey').clearValidators(); + break; + case Lwm2mSecurityType.PSK: + this.lenMinClientPublicKeyOrId = 0; + this.lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; + this.lengthClientSecretKey = LEN_MAX_PSK; + this.setValidatorsSecurity(securityMode); + break; + case Lwm2mSecurityType.RPK: + this.lenMinClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; + this.lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; + this.lengthClientSecretKey = LEN_MAX_PRIVATE_KEY; + this.setValidatorsSecurity(securityMode); + break; + case Lwm2mSecurityType.X509: + this.lenMinClientPublicKeyOrId = 0; + this.lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_X509; + this.lengthClientSecretKey = LEN_MAX_PRIVATE_KEY; + this.setValidatorsSecurity(securityMode); + break; + } + this.serverFormGroup.get('clientPublicKeyOrId').updateValueAndValidity({emitEvent: false}); + this.serverFormGroup.get('clientSecretKey').updateValueAndValidity({emitEvent: !initValue}); + } + + private setValidatorsSecurity = (securityMode: Lwm2mSecurityType): void => { + if (securityMode === Lwm2mSecurityType.PSK) { + this.serverFormGroup.get('clientPublicKeyOrId').setValidators([Validators.required]); + } else { + this.serverFormGroup.get('clientPublicKeyOrId').setValidators([ + Validators.required, + Validators.pattern(KEY_REGEXP_HEX_DEC), + Validators.minLength(this.lenMinClientPublicKeyOrId), + Validators.maxLength(this.lenMaxClientPublicKeyOrId) + ]); + } + + this.serverFormGroup.get('clientSecretKey').setValidators([ + Validators.required, + Validators.pattern(KEY_REGEXP_HEX_DEC), + Validators.minLength(this.lengthClientSecretKey), + Validators.maxLength(this.lengthClientSecretKey) + ]); + } +} diff --git a/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.html b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.html new file mode 100644 index 0000000000..bd83cc817b --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.html @@ -0,0 +1,143 @@ + +
+ +

{{ title }}

+ + + +
+ + device.lwm2m-security-config.endpoint + + + {{ 'device.lwm2m-security-config.endpoint-required' | translate }} + + + + + + device.lwm2m-security-config.mode + + + {{ credentialTypeLwM2MNamesMap.get(securityConfigLwM2MType[securityConfigClientMode]) }} + + + +
+ + {{ 'device.lwm2m-security-config.identity' | translate }} + + + {{ 'device.lwm2m-security-config.identity-required' | translate }} + + +
+ + {{ 'device.lwm2m-security-config.client-key' | translate }} + + {{key.value?.length || 0}}/{{lenMaxKeyClient}} + + {{ 'device.lwm2m-security-config.client-key-required' | translate }} + + + {{ 'device.lwm2m-security-config.client-key-pattern' | translate }} + + + {{ 'device.lwm2m-security-config.client-key-length' | translate: { + count: lenMaxKeyClient + } }} + + + + {{ 'device.lwm2m-security-config.client-certificate' | translate }} + + + +
+ + + + + {{ 'device.lwm2m-security-config.bootstrap-server' | translate }} + + + + + + + + + + + {{ 'device.lwm2m-security-config.lwm2m-server' | translate }} + + + + + + + + +
+ + + + + + + + +
+
+ + +
+
diff --git a/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.scss b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.scss new file mode 100644 index 0000000000..0188fcd559 --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.scss @@ -0,0 +1,34 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +:host { + mat-tab-group { + min-height: 330px; + } +} + +:host ::ng-deep { + .mat-tab-body-wrapper { + min-height: 200px; + } + + .mat-tab-body { + padding: 16px 0; + } + + textarea.mat-input-element.cdk-textarea-autosize { + box-sizing: content-box; + } +} diff --git a/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.ts b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.ts new file mode 100644 index 0000000000..419576753c --- /dev/null +++ b/ui-ngx/src/app/modules/home/components/device/security-config-lwm2m.component.ts @@ -0,0 +1,257 @@ +/// +/// Copyright © 2016-2021 The Thingsboard Authors +/// +/// Licensed under the Apache License, Version 2.0 (the "License"); +/// you may not use this file except in compliance with the License. +/// You may obtain a copy of the License at +/// +/// http://www.apache.org/licenses/LICENSE-2.0 +/// +/// Unless required by applicable law or agreed to in writing, software +/// distributed under the License is distributed on an "AS IS" BASIS, +/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +/// See the License for the specific language governing permissions and +/// limitations under the License. +/// + + +import { Component, Inject, OnDestroy, OnInit } from '@angular/core'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { FormBuilder, FormGroup, Validators } from '@angular/forms'; +import { TranslateService } from '@ngx-translate/core'; +import { + DeviceCredentialsDialogLwm2mData, + getClientSecurityConfig, + JSON_ALL_CONFIG, + KEY_REGEXP_HEX_DEC, + LEN_MAX_PSK, + LEN_MAX_PUBLIC_KEY_RPK, + Lwm2mSecurityConfigModels, + Lwm2mSecurityType, + Lwm2mSecurityTypeTranslationMap +} from '@shared/models/lwm2m-security-config.models'; +import { MatTabChangeEvent } from '@angular/material/tabs'; +import { MatTab } from '@angular/material/tabs/tab'; +import { Subject } from 'rxjs'; +import { takeUntil } from 'rxjs/operators'; + +@Component({ + selector: 'tb-security-config-lwm2m', + templateUrl: './security-config-lwm2m.component.html', + styleUrls: ['./security-config-lwm2m.component.scss'] +}) + +export class SecurityConfigLwm2mComponent extends DialogComponent implements OnInit, OnDestroy { + + private destroy$ = new Subject(); + + lwm2mConfigFormGroup: FormGroup; + title: string; + securityConfigLwM2MType = Lwm2mSecurityType; + securityConfigLwM2MTypes = Object.keys(Lwm2mSecurityType); + credentialTypeLwM2MNamesMap = Lwm2mSecurityTypeTranslationMap; + formControlNameJsonAllConfig = JSON_ALL_CONFIG; + jsonAllConfig: Lwm2mSecurityConfigModels; + lenMaxKeyClient = LEN_MAX_PSK; + tabPrevious: MatTab; + tabIndexPrevious = 0; + + constructor(protected store: Store, + protected router: Router, + @Inject(MAT_DIALOG_DATA) public data: DeviceCredentialsDialogLwm2mData, + public dialogRef: MatDialogRef, + public fb: FormBuilder, + public translate: TranslateService) { + super(store, router, dialogRef); + } + + ngOnInit() { + this.jsonAllConfig = JSON.parse(JSON.stringify(this.data.jsonAllConfig)); + this.lwm2mConfigFormGroup = this.initLwm2mConfigFormGroup(); + this.title = this.translate.instant('device.lwm2m-security-info') + ': ' + this.data.endPoint; + this.lwm2mConfigFormGroup.get('x509').disable(); + this.initClientSecurityConfig(this.lwm2mConfigFormGroup.get('jsonAllConfig').value); + this.registerDisableOnLoadFormControl(this.lwm2mConfigFormGroup.get('securityConfigClientMode')); + } + + ngOnDestroy() { + this.destroy$.next(); + this.destroy$.complete(); + } + + private initClientSecurityConfig = (jsonAllConfig: Lwm2mSecurityConfigModels): void => { + if (jsonAllConfig.client.securityConfigClientMode !== Lwm2mSecurityType.NO_SEC) { + this.lwm2mConfigFormGroup.patchValue(jsonAllConfig.client, {emitEvent: false}); + } + this.securityConfigClientUpdateValidators(jsonAllConfig.client.securityConfigClientMode); + } + + private securityConfigClientModeChanged(type: Lwm2mSecurityType): void { + const config = getClientSecurityConfig(type, this.lwm2mConfigFormGroup.get('endPoint').value); + switch (type) { + case Lwm2mSecurityType.PSK: + config.identity = this.data.endPoint; + config.key = this.lwm2mConfigFormGroup.get('key').value; + break; + case Lwm2mSecurityType.RPK: + config.key = this.lwm2mConfigFormGroup.get('key').value; + break; + } + this.jsonAllConfig.client = config; + this.lwm2mConfigFormGroup.patchValue({ + ...config, + jsonAllConfig: this.jsonAllConfig + }, {emitEvent: false}); + this.securityConfigClientUpdateValidators(type); + } + + private securityConfigClientUpdateValidators = (mode: Lwm2mSecurityType): void => { + switch (mode) { + case Lwm2mSecurityType.NO_SEC: + case Lwm2mSecurityType.X509: + this.setValidatorsNoSecX509(); + break; + case Lwm2mSecurityType.PSK: + this.lenMaxKeyClient = LEN_MAX_PSK; + this.setValidatorsPskRpk(mode); + break; + case Lwm2mSecurityType.RPK: + this.lenMaxKeyClient = LEN_MAX_PUBLIC_KEY_RPK; + this.setValidatorsPskRpk(mode); + break; + } + this.lwm2mConfigFormGroup.get('identity').updateValueAndValidity({emitEvent: false}); + this.lwm2mConfigFormGroup.get('key').updateValueAndValidity({emitEvent: false}); + } + + private setValidatorsNoSecX509 = (): void => { + this.lwm2mConfigFormGroup.get('identity').setValidators([]); + this.lwm2mConfigFormGroup.get('key').setValidators([]); + } + + private setValidatorsPskRpk = (mode: Lwm2mSecurityType): void => { + if (mode === Lwm2mSecurityType.PSK) { + this.lwm2mConfigFormGroup.get('identity').setValidators([Validators.required]); + } else { + this.lwm2mConfigFormGroup.get('identity').setValidators([]); + } + this.lwm2mConfigFormGroup.get('key').setValidators([Validators.required, + Validators.pattern(KEY_REGEXP_HEX_DEC), + Validators.maxLength(this.lenMaxKeyClient), Validators.minLength(this.lenMaxKeyClient)]); + } + + tabChanged = (tabChangeEvent: MatTabChangeEvent): void => { + if (this.tabIndexPrevious !== tabChangeEvent.index) { + this.upDateValueToJson(); + } + this.tabIndexPrevious = tabChangeEvent.index; + } + + private upDateValueToJson(): void { + switch (this.tabIndexPrevious) { + case 0: + this.upDateValueToJsonTab0(); + break; + case 1: + this.upDateValueToJsonTab1(); + break; + } + } + + private upDateValueToJsonTab0 = (): void => { + if (this.lwm2mConfigFormGroup.get('identity').dirty && this.lwm2mConfigFormGroup.get('identity').valid || + this.lwm2mConfigFormGroup.get('key').dirty && this.lwm2mConfigFormGroup.get('key').valid) { + this.updateBootstrapSettings(); + this.upDateJsonAllConfig(); + } + } + + private upDateValueToJsonTab1 = (): void => { + const bootstrap = this.lwm2mConfigFormGroup.get('bootstrapServer').value; + if (bootstrap !== null + && this.lwm2mConfigFormGroup.get('bootstrapServer').dirty + && this.lwm2mConfigFormGroup.get('bootstrapServer').valid) { + this.jsonAllConfig.bootstrap.bootstrapServer = bootstrap; + this.upDateJsonAllConfig(); + } + const serverConfig = this.lwm2mConfigFormGroup.get('lwm2mServer').value; + if (serverConfig !== null + && this.lwm2mConfigFormGroup.get('lwm2mServer').dirty + && this.lwm2mConfigFormGroup.get('lwm2mServer').valid) { + this.jsonAllConfig.bootstrap.lwm2mServer = serverConfig; + this.upDateJsonAllConfig(); + } + } + + private updateBootstrapSettings() { + const securityMode = 'securityMode'; + this.jsonAllConfig.client.identity = this.lwm2mConfigFormGroup.get('identity').value; + this.jsonAllConfig.client.key = this.lwm2mConfigFormGroup.get('key').value; + if (this.lwm2mConfigFormGroup.get('bootstrapServer').value[securityMode] === Lwm2mSecurityType.PSK) { + this.jsonAllConfig.bootstrap.bootstrapServer.clientPublicKeyOrId = this.jsonAllConfig.client.identity; + this.jsonAllConfig.bootstrap.bootstrapServer.clientSecretKey = this.jsonAllConfig.client.key; + this.lwm2mConfigFormGroup.get('bootstrapServer').patchValue(this.jsonAllConfig.bootstrap.bootstrapServer, {emitEvent: false}); + } + if (this.lwm2mConfigFormGroup.get('lwm2mServer').value[securityMode] === Lwm2mSecurityType.PSK) { + this.jsonAllConfig.bootstrap.lwm2mServer.clientPublicKeyOrId = this.jsonAllConfig.client.identity; + this.jsonAllConfig.bootstrap.lwm2mServer.clientSecretKey = this.jsonAllConfig.client.key; + this.lwm2mConfigFormGroup.get('lwm2mServer').patchValue(this.jsonAllConfig.bootstrap.lwm2mServer, {emitEvent: false}); + } + } + + private upDateJsonAllConfig = (): void => { + this.lwm2mConfigFormGroup.patchValue({ + jsonAllConfig: this.jsonAllConfig + }, {emitEvent: false}); + } + + private initLwm2mConfigFormGroup = (): FormGroup => { + if (this.jsonAllConfig.client.securityConfigClientMode === Lwm2mSecurityType.PSK) { + this.data.endPoint = this.jsonAllConfig.client.endpoint; + } + const formGroup = this.fb.group({ + securityConfigClientMode: [this.jsonAllConfig.client.securityConfigClientMode], + identity: [''], + key: [''], + x509: [false], + bootstrapServer: [this.jsonAllConfig.bootstrap.bootstrapServer], + lwm2mServer: [this.jsonAllConfig.bootstrap.lwm2mServer], + endPoint: [this.data.endPoint], + jsonAllConfig: [this.jsonAllConfig] + }); + formGroup.get('securityConfigClientMode').valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe((type) => { + this.securityConfigClientModeChanged(type); + }); + formGroup.get('endPoint').valueChanges.pipe( + takeUntil(this.destroy$) + ).subscribe((endpoint) => { + if (formGroup.get('securityConfigClientMode').value === Lwm2mSecurityType.PSK) { + this.jsonAllConfig.client.endpoint = endpoint; + this.upDateJsonAllConfig(); + } + }); + return formGroup; + } + + save(): void { + this.upDateValueToJson(); + this.data.endPoint = this.lwm2mConfigFormGroup.get('endPoint').value.split('\'').join(''); + this.data.jsonAllConfig = this.jsonAllConfig; + if (this.lwm2mConfigFormGroup.get('securityConfigClientMode').value === Lwm2mSecurityType.PSK) { + this.data.endPoint = this.data.jsonAllConfig.client.identity; + } + this.dialogRef.close(this.data); + } + + cancel(): void { + this.dialogRef.close(undefined); + } +} + + diff --git a/ui-ngx/src/app/modules/home/components/home-components.module.ts b/ui-ngx/src/app/modules/home/components/home-components.module.ts index f2208757d4..a5a3ed460b 100644 --- a/ui-ngx/src/app/modules/home/components/home-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/home-components.module.ts @@ -138,6 +138,8 @@ import { EMBED_DASHBOARD_DIALOG_TOKEN } from '@home/components/widget/dialog/emb import { EdgeDownlinkTableComponent } from '@home/components/edge/edge-downlink-table.component'; import { EdgeDownlinkTableHeaderComponent } from '@home/components/edge/edge-downlink-table-header.component'; import { DisplayWidgetTypesPanelComponent } from '@home/components/dashboard-page/widget-types-panel.component'; +import { SecurityConfigLwm2mComponent } from '@home/components/device/security-config-lwm2m.component'; +import { SecurityConfigLwm2mServerComponent } from '@home/components/device/security-config-lwm2m-server.component'; @NgModule({ declarations: @@ -239,6 +241,8 @@ import { DisplayWidgetTypesPanelComponent } from '@home/components/dashboard-pag DeviceWizardDialogComponent, DeviceCredentialsComponent, CopyDeviceCredentialsComponent, + SecurityConfigLwm2mComponent, + SecurityConfigLwm2mServerComponent, AlarmScheduleDialogComponent, EditAlarmDetailsDialogComponent, SmsProviderConfigurationComponent, @@ -342,6 +346,8 @@ import { DisplayWidgetTypesPanelComponent } from '@home/components/dashboard-pag DeviceWizardDialogComponent, DeviceCredentialsComponent, CopyDeviceCredentialsComponent, + SecurityConfigLwm2mComponent, + SecurityConfigLwm2mServerComponent, AlarmScheduleInfoComponent, AlarmScheduleComponent, AlarmScheduleDialogComponent, diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-dialog.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-dialog.component.ts index 834be7535c..c1877e218e 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-dialog.component.ts @@ -14,21 +14,20 @@ /// limitations under the License. /// -import {Component, Inject, OnInit, SkipSelf} from "@angular/core"; -import {ErrorStateMatcher} from "@angular/material/core"; -import {DialogComponent} from "@shared/components/dialog.component"; -import {Store} from "@ngrx/store"; -import {AppState} from "@core/core.state"; -import {Router} from "@angular/router"; -import {MAT_DIALOG_DATA, MatDialogRef} from "@angular/material/dialog"; -import {FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm} from "@angular/forms"; -import {TranslateService} from "@ngx-translate/core"; -import {JsonObject} from "@angular/compiler-cli/ngcc/src/packages/entry_point"; +import { Component, Inject, OnInit, SkipSelf } from '@angular/core'; +import { ErrorStateMatcher } from '@angular/material/core'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; +import { FormBuilder, FormControl, FormGroup, FormGroupDirective, NgForm } from '@angular/forms'; +import { JsonObject } from '@angular/compiler-cli/ngcc/src/packages/entry_point'; export interface Lwm2mAttributesDialogData { readonly: boolean; attributeLwm2m: JsonObject; - destName: string + destName: string; } @Component({ @@ -37,7 +36,8 @@ export interface Lwm2mAttributesDialogData { styleUrls: ['./lwm2m-attributes.component.scss'], providers: [{provide: ErrorStateMatcher, useExisting: Lwm2mAttributesDialogComponent}], }) -export class Lwm2mAttributesDialogComponent extends DialogComponent implements OnInit, ErrorStateMatcher { +export class Lwm2mAttributesDialogComponent extends DialogComponent + implements OnInit, ErrorStateMatcher { readonly = this.data.readonly; @@ -54,8 +54,7 @@ export class Lwm2mAttributesDialogComponent extends DialogComponent, - private fb: FormBuilder, - public translate: TranslateService) { + private fb: FormBuilder) { super(store, router, dialogRef); this.attributeLwm2mDialogFormGroup = this.fb.group({ diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-key-list.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-key-list.component.ts index 7d7af196f0..686382e9bd 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-key-list.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes-key-list.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {Component, forwardRef, Input, OnInit} from "@angular/core"; +import { Component, forwardRef, Input, OnInit } from '@angular/core'; import { AbstractControl, ControlValueAccessor, @@ -26,17 +26,17 @@ import { NG_VALUE_ACCESSOR, Validator, Validators -} from "@angular/forms"; -import {Subscription} from "rxjs"; -import {PageComponent} from "@shared/components/page.component"; -import {Store} from "@ngrx/store"; -import {AppState} from "@core/core.state"; +} from '@angular/forms'; +import { Subscription } from 'rxjs'; import { ATTRIBUTE_KEYS, ATTRIBUTE_LWM2M_ENUM, ATTRIBUTE_LWM2M_MAP -} from "@home/components/profile/device/lwm2m/lwm2m-profile-config.models"; -import {isDefinedAndNotNull, isEmpty, isEmptyStr, isUndefinedOrNull} from "@core/utils"; +} from './lwm2m-profile-config.models'; +import { isDefinedAndNotNull, isEmpty, isEmptyStr, isUndefinedOrNull } from '@core/utils'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { PageComponent } from '@shared/components/page.component'; @Component({ @@ -98,7 +98,7 @@ export class Lwm2mAttributesKeyListComponent extends PageComponent implements Co registerOnTouched(fn: any): void { } - setDisabledState?(isDisabled: boolean): void { + setDisabledState(isDisabled: boolean): void { this.disabled = isDisabled; if (this.disabled) { this.kvListFormGroup.disable({emitEvent: false}); @@ -166,10 +166,10 @@ export class Lwm2mAttributesKeyListComponent extends PageComponent implements Co }; } - private updateValidate (c?: FormControl) { + private updateValidate() { const kvList = this.kvListFormGroup.get('keyVals') as FormArray; kvList.controls.forEach(fg => { - if (fg.get('key').value==='ver') { + if (fg.get('key').value === 'ver') { fg.get('value').setValidators(null); fg.get('value').setErrors(null); } @@ -189,7 +189,7 @@ export class Lwm2mAttributesKeyListComponent extends PageComponent implements Co if (isUndefinedOrNull(entry.value) || entry.key === 'ver' || isEmptyStr(entry.value.toString())) { keyValMap[entry.key] = entry.value.toString(); } else { - keyValMap[entry.key] = Number(entry.value) + keyValMap[entry.key] = Number(entry.value); } }); this.propagateChange(keyValMap); @@ -215,13 +215,13 @@ export class Lwm2mAttributesKeyListComponent extends PageComponent implements Co private attributeLwm2mValueNumberValidator = (control: AbstractControl) => { if (isNaN(Number(control.value)) || Number(control.value) < 0) { return { - 'validAttributeValue': true + validAttributeValue: true }; } return null; } - private attributeLwm2mValueValidator = (property: string): Object [] => { - return property === 'ver'? [] : [this.attributeLwm2mValueNumberValidator]; + private attributeLwm2mValueValidator = (property: string): object[] => { + return property === 'ver' ? [] : [this.attributeLwm2mValueNumberValidator]; } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes.component.ts index ca4b66ae36..1e467bcd93 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-attributes.component.ts @@ -14,21 +14,14 @@ /// limitations under the License. /// -import {Component, EventEmitter, forwardRef, Inject, Input, Output} from "@angular/core"; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR} from "@angular/forms"; -import {coerceBooleanProperty} from "@angular/cdk/coercion"; -import {Store} from "@ngrx/store"; -import {AppState} from "@core/core.state"; -import {DeviceProfileService} from "@core/http/device-profile.service"; -import {WINDOW} from "@core/services/window.service"; -import {deepClone, isDefinedAndNotNull, isEmpty} from "@core/utils"; -import { - Lwm2mAttributesDialogComponent, - Lwm2mAttributesDialogData -} from "@home/components/profile/device/lwm2m/lwm2m-attributes-dialog.component"; -import {MatDialog} from "@angular/material/dialog"; -import {TranslateService} from "@ngx-translate/core"; -import {ATTRIBUTE_LWM2M_LABEL} from "@home/components/profile/device/lwm2m/lwm2m-profile-config.models"; +import { Component, EventEmitter, forwardRef, Input, Output } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR } from '@angular/forms'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { deepClone, isDefinedAndNotNull, isEmpty } from '@core/utils'; +import { Lwm2mAttributesDialogComponent, Lwm2mAttributesDialogData } from './lwm2m-attributes-dialog.component'; +import { MatDialog } from '@angular/material/dialog'; +import { TranslateService } from '@ngx-translate/core'; +import { ATTRIBUTE_LWM2M_LABEL } from './lwm2m-profile-config.models'; @Component({ @@ -46,7 +39,6 @@ export class Lwm2mAttributesComponent implements ControlValueAccessor { attributeLwm2mLabel = ATTRIBUTE_LWM2M_LABEL; private requiredValue: boolean; - private dirty = false; @Input() attributeLwm2m: {}; @@ -71,12 +63,9 @@ export class Lwm2mAttributesComponent implements ControlValueAccessor { private propagateChange = (v: any) => { } - constructor(private store: Store, - private dialog: MatDialog, + constructor(private dialog: MatDialog, private fb: FormBuilder, - private deviceProfileService: DeviceProfileService, - private translate: TranslateService, - @Inject(WINDOW) private window: Window) {} + private translate: TranslateService) {} registerOnChange(fn: any): void { this.propagateChange = fn; @@ -117,20 +106,20 @@ export class Lwm2mAttributesComponent implements ControlValueAccessor { return label; } - isDisableBtn (): boolean { + isDisableBtn(): boolean { return this.disabled || this.isAttributeTelemetry ? !(isDefinedAndNotNull(this.attributeLwm2m) && !isEmpty(this.attributeLwm2m) && this.disabled) : this.disabled; } - isIconView (): boolean { + isIconView(): boolean { return this.isAttributeTelemetry || this.disabled; } - isIconEditAdd (): boolean { + isIconEditAdd(): boolean { return isDefinedAndNotNull(this.attributeLwm2m) && !isEmpty(this.attributeLwm2m); } - isToolTipLabel (): string { + isToolTipLabel(): string { return this.disabled ? this.translate.instant('device-profile.lwm2m.attribute-lwm2m-tip') : this.isAttributeTelemetry ? this.translate.instant('device-profile.lwm2m.attribute-lwm2m-disable-tip') : this.translate.instant('device-profile.lwm2m.attribute-lwm2m-tip'); @@ -140,7 +129,7 @@ export class Lwm2mAttributesComponent implements ControlValueAccessor { if ($event) { $event.stopPropagation(); } - this.dialog.open(Lwm2mAttributesDialogComponent, { + this.dialog.open(Lwm2mAttributesDialogComponent, { disableClose: true, panelClass: ['tb-dialog', 'tb-fullscreen-dialog'], data: { diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts index b82da4bc06..55f8db8f84 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-config-server.component.ts @@ -14,8 +14,8 @@ /// limitations under the License. /// -import {Component, forwardRef, Inject, Input} from '@angular/core'; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; +import { Component, forwardRef, Inject, Input } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; import { DEFAULT_CLIENT_HOLD_OFF_TIME, DEFAULT_ID_SERVER, @@ -28,13 +28,12 @@ import { SECURITY_CONFIG_MODE_NAMES, ServerSecurityConfig } from './lwm2m-profile-config.models'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; -import {WINDOW} from '@core/services/window.service'; -import {pairwise, startWith} from 'rxjs/operators'; -import {DeviceProfileService} from '@core/http/device-profile.service'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { WINDOW } from '@core/services/window.service'; +import { pairwise, startWith } from 'rxjs/operators'; +import { DeviceProfileService } from '@core/http/device-profile.service'; +// @dynamic @Component({ selector: 'tb-profile-lwm2m-device-config-server', templateUrl: './lwm2m-device-config-server.component.html', @@ -73,8 +72,7 @@ export class Lwm2mDeviceConfigServerComponent implements ControlValueAccessor { this.requiredValue = coerceBooleanProperty(value); } - constructor(protected store: Store, - public fb: FormBuilder, + constructor(public fb: FormBuilder, private deviceProfileService: DeviceProfileService, @Inject(WINDOW) private window: Window) { this.serverFormGroup = this.initServerGroup(); diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts index d6f6823db6..60fb27a752 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-device-profile-transport-configuration.component.ts @@ -15,10 +15,8 @@ /// import { DeviceProfileTransportConfiguration } from '@shared/models/device.models'; -import { Component, forwardRef, Inject, Input } from '@angular/core'; +import { Component, forwardRef, Input } from '@angular/core'; import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; -import { Store } from '@ngrx/store'; -import { AppState } from '@app/core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { ATTRIBUTE, @@ -37,7 +35,6 @@ import { } from './lwm2m-profile-config.models'; import { DeviceProfileService } from '@core/http/device-profile.service'; import { deepClone, isDefinedAndNotNull, isEmpty, isUndefined } from '@core/utils'; -import { WINDOW } from '@core/services/window.service'; import { JsonArray, JsonObject } from '@angular/compiler-cli/ngcc/src/packages/entry_point'; import { Direction } from '@shared/models/page/sort-order'; @@ -78,10 +75,8 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro private propagateChange = (v: any) => { } - constructor(private store: Store, - private fb: FormBuilder, - private deviceProfileService: DeviceProfileService, - @Inject(WINDOW) private window: Window) { + constructor(private fb: FormBuilder, + private deviceProfileService: DeviceProfileService) { this.lwm2mDeviceProfileFormGroup = this.fb.group({ clientOnlyObserveAfterConnect: [1, []], objectIds: [null, Validators.required], @@ -155,7 +150,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro this.lwm2mDeviceProfileFormGroup.patchValue({ clientOnlyObserveAfterConnect: this.configurationValue.clientLwM2mSettings.clientOnlyObserveAfterConnect, objectIds: value, - observeAttrTelemetry: this.getObserveAttrTelemetryObjects(value['objectsList']), + observeAttrTelemetry: this.getObserveAttrTelemetryObjects(value.objectsList), shortId: this.configurationValue.bootstrap.servers.shortId, lifetime: this.configurationValue.bootstrap.servers.lifetime, defaultMinPeriod: this.configurationValue.bootstrap.servers.defaultMinPeriod, @@ -171,7 +166,6 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro let configuration: DeviceProfileTransportConfiguration = null; if (this.lwm2mDeviceConfigFormGroup.valid && this.lwm2mDeviceProfileFormGroup.valid) { configuration = this.lwm2mDeviceConfigFormGroup.value.configurationJson; - // configuration.type = DeviceTransportType.LWM2M; } this.propagateChange(configuration); } @@ -206,7 +200,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro if (this.configurationValue.observeAttr && objectLwM2MS.length > 0) { const attributeArray = this.configurationValue.observeAttr.attribute; const telemetryArray = this.configurationValue.observeAttr.telemetry; - let keyNameJson = this.configurationValue.observeAttr.keyName; + const keyNameJson = this.configurationValue.observeAttr.keyName; if (this.includesNotZeroInstance(attributeArray, telemetryArray)) { this.addInstances(attributeArray, telemetryArray, objectLwM2MS); } @@ -270,9 +264,9 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro private updateAttributeLwm2m = (objectLwM2MS: ObjectLwM2M[]): void => { Object.keys(this.configurationValue.observeAttr.attributeLwm2m).forEach(key => { const [objectKeyId, instanceId, resourceId] = Array.from(key.substring(1).split('/'), String); - let objectLwM2M = objectLwM2MS.find(objectLwm2m => objectLwm2m.keyId === objectKeyId); + const objectLwM2M = objectLwM2MS.find(objectLwm2m => objectLwm2m.keyId === objectKeyId); if (objectLwM2M && instanceId) { - let instance = objectLwM2M.instances.find(instance => instance.id === +instanceId) + const instance = objectLwM2M.instances.find(obj => obj.id === +instanceId); if (instance && resourceId) { instance.resources.find(resource => resource.id === +resourceId) .attributeLwm2m = this.configurationValue.observeAttr.attributeLwm2m[key]; @@ -288,7 +282,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro private updateKeyNameObjects = (objectLwM2MS: ObjectLwM2M[]): void => { Object.keys(this.configurationValue.observeAttr.keyName).forEach(key => { const [objectKeyId, instanceId, resourceId] = Array.from(key.substring(1).split('/'), String); - const objectLwM2M = objectLwM2MS.find(objectLwm2m => objectLwm2m.keyId === objectKeyId) + const objectLwM2M = objectLwM2MS.find(objectLwm2m => objectLwm2m.keyId === objectKeyId); if (objectLwM2M) { objectLwM2M.instances.find(instance => instance.id === +instanceId) .resources.find(resource => resource.id === +resourceId) @@ -299,7 +293,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro private validateKeyNameObjects = (nameJson: JsonObject, attributeArray: JsonArray, telemetryArray: JsonArray): {} => { const keyName = JSON.parse(JSON.stringify(nameJson)); - let keyNameValidate = {}; + const keyNameValidate = {}; const keyAttrTelemetry = attributeArray.concat(telemetryArray); Object.keys(keyName).forEach(key => { if (keyAttrTelemetry.includes(key)) { @@ -345,9 +339,9 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro attributeLwm2m[pathRes] = resource.attributeLwm2m; } } - }) + }); } - }) + }); } }); if (isUndefined(this.configurationValue.observeAttr)) { @@ -356,7 +350,7 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro attribute: attributeArray, telemetry: telemetryArray, keyName: this.sortObjectKeyPathJson(KEY_NAME, keyNameNew), - attributeLwm2m: attributeLwm2m + attributeLwm2m }; } else { this.configurationValue.observeAttr.observe = observeArray; @@ -457,7 +451,6 @@ export class Lwm2mDeviceProfileTransportConfigurationComponent implements Contro } private removeAttributeLwm2mFromJson = (keyId: string): void => { - debugger const keyNameJson = this.configurationValue.observeAttr.attributeLwm2m; Object.keys(keyNameJson).forEach(key => { if (key.startsWith(`/${keyId}`)) { diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-dialog.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-dialog.component.ts index c480356e72..030b19b6d2 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-dialog.component.ts @@ -14,13 +14,13 @@ /// limitations under the License. /// -import {Component, Inject, OnInit} from '@angular/core'; -import {DialogComponent} from '@shared/components/dialog.component'; -import {FormBuilder, FormGroup} from '@angular/forms'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {Router} from '@angular/router'; -import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material/dialog'; +import { Component, Inject, OnInit } from '@angular/core'; +import { DialogComponent } from '@shared/components/dialog.component'; +import { FormBuilder, FormGroup } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; +import { Router } from '@angular/router'; +import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; export interface Lwm2mObjectAddInstancesData { instancesIds: Set; diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-list.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-list.component.ts index cda075a142..29f5bb33cf 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-list.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-add-instances-list.component.ts @@ -14,11 +14,9 @@ /// limitations under the License. /// -import {Component, forwardRef} from '@angular/core'; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {INSTANCES_ID_VALUE_MAX, INSTANCES_ID_VALUE_MIN, KEY_REGEXP_NUMBER} from './lwm2m-profile-config.models'; +import { Component, forwardRef } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { INSTANCES_ID_VALUE_MAX, INSTANCES_ID_VALUE_MIN, KEY_REGEXP_NUMBER } from './lwm2m-profile-config.models'; @Component({ selector: 'tb-profile-lwm2m-object-add-instances-list', @@ -41,8 +39,7 @@ export class Lwm2mObjectAddInstancesListComponent implements ControlValueAccesso private propagateChange = (v: any) => { }; - constructor(private store: Store, - private fb: FormBuilder) { + constructor(private fb: FormBuilder) { this.instancesListFormGroup = this.fb.group({ instanceIdInput: [null, [ Validators.min(this.instanceIdValueMin), diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts index ba54f63411..8f1f676769 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-object-list.component.ts @@ -14,18 +14,16 @@ /// limitations under the License. /// -import {Component, ElementRef, EventEmitter, forwardRef, Input, OnInit, Output, ViewChild} from '@angular/core'; -import {ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {Observable} from 'rxjs'; -import {filter, map, mergeMap, publishReplay, refCount, tap} from 'rxjs/operators'; -import {ModelValue, ObjectLwM2M, PAGE_SIZE_LIMIT} from './lwm2m-profile-config.models'; -import {DeviceProfileService} from '@core/http/device-profile.service'; -import {Direction} from '@shared/models/page/sort-order'; -import {isDefined, isDefinedAndNotNull, isString} from '@core/utils'; -import {PageLink} from "@shared/models/page/page-link"; +import { Component, ElementRef, EventEmitter, forwardRef, Input, OnInit, Output, ViewChild } from '@angular/core'; +import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; +import { Observable } from 'rxjs'; +import { filter, map, mergeMap, publishReplay, refCount, tap } from 'rxjs/operators'; +import { ModelValue, ObjectLwM2M, PAGE_SIZE_LIMIT } from './lwm2m-profile-config.models'; +import { DeviceProfileService } from '@core/http/device-profile.service'; +import { Direction } from '@shared/models/page/sort-order'; +import { isDefined, isDefinedAndNotNull, isString } from '@core/utils'; +import { PageLink } from '@shared/models/page/page-link'; @Component({ selector: 'tb-profile-lwm2m-object-list', @@ -71,8 +69,7 @@ export class Lwm2mObjectListComponent implements ControlValueAccessor, OnInit, V private propagateChange = (v: any) => { } - constructor(private store: Store, - private deviceProfileService: DeviceProfileService, + constructor(private deviceProfileService: DeviceProfileService, private fb: FormBuilder) { this.lwm2mListFormGroup = this.fb.group({ objectsList: [this.objectsList], @@ -162,20 +159,20 @@ export class Lwm2mObjectListComponent implements ControlValueAccessor, OnInit, V private fetchListObjects = (searchText?: string): Observable> => { this.searchText = searchText; - return this.getLwM2mModelsPage().pipe( - map(objectLwM2Ms => objectLwM2Ms) - ); + return this.getLwM2mModelsPage().pipe( + map(objectLwM2Ms => objectLwM2Ms) + ); } private getLwM2mModelsPage(): Observable> { - const pageLink = new PageLink(PAGE_SIZE_LIMIT, 0, this.searchText, { - property: 'id', - direction: Direction.ASC - }); - this.lw2mModels = this.deviceProfileService.getLwm2mObjectsPage(pageLink).pipe( - publishReplay(1), - refCount() - ); + const pageLink = new PageLink(PAGE_SIZE_LIMIT, 0, this.searchText, { + property: 'id', + direction: Direction.ASC + }); + this.lw2mModels = this.deviceProfileService.getLwm2mObjectsPage(pageLink).pipe( + publishReplay(1), + refCount() + ); return this.lw2mModels; } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry-resource.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry-resource.component.ts index c824b000a8..c3003c7c3e 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry-resource.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry-resource.component.ts @@ -14,13 +14,11 @@ /// limitations under the License. /// -import {Component, forwardRef, Input} from '@angular/core'; -import {ControlValueAccessor, FormArray, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators} from '@angular/forms'; -import {ResourceLwM2M, RESOURCES} from '@home/components/profile/device/lwm2m/lwm2m-profile-config.models'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; +import { Component, forwardRef, Input } from '@angular/core'; +import { ControlValueAccessor, FormArray, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { ResourceLwM2M, RESOURCES } from '@home/components/profile/device/lwm2m/lwm2m-profile-config.models'; import _ from 'lodash'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; @Component({ selector: 'tb-profile-lwm2m-observe-attr-telemetry-resource', @@ -54,8 +52,7 @@ export class Lwm2mObserveAttrTelemetryResourceComponent implements ControlValueA } } - constructor(private store: Store, - private fb: FormBuilder) { + constructor(private fb: FormBuilder) { this.resourceFormGroup = this.fb.group({ resources: this.fb.array([]) }); @@ -95,7 +92,7 @@ export class Lwm2mObserveAttrTelemetryResourceComponent implements ControlValueA } getNameResourceLwm2m = (resourceLwM2M: ResourceLwM2M): string => { - return '<' + resourceLwM2M.id +'> ' + resourceLwM2M.name; + return `<${resourceLwM2M.id}> ${resourceLwM2M.name}`; } createResourceLwM2M(resourcesLwM2M: ResourceLwM2M[]): void { @@ -135,14 +132,14 @@ export class Lwm2mObserveAttrTelemetryResourceComponent implements ControlValueA return index; } - updateObserve = (index: number): void =>{ + updateObserve = (index: number): void => { if (this.resourceFormArray.at(index).value.attribute === false && this.resourceFormArray.at(index).value.telemetry === false) { this.resourceFormArray.at(index).patchValue({observe: false}); this.resourceFormArray.at(index).patchValue({attributeLwm2m: {}}); } } - disableObserve = (index: number): boolean =>{ + disableObserve = (index: number): boolean => { return !this.resourceFormArray.at(index).value.telemetry && !this.resourceFormArray.at(index).value.attribute; } } diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts index c38c413e98..77a5956855 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts @@ -14,7 +14,7 @@ /// limitations under the License. /// -import {Component, forwardRef, Input} from '@angular/core'; +import { Component, forwardRef, Input } from '@angular/core'; import { AbstractControl, ControlValueAccessor, @@ -24,9 +24,7 @@ import { NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {coerceBooleanProperty} from '@angular/cdk/coercion'; +import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { ATTRIBUTE, ATTRIBUTE_LWM2M, @@ -38,9 +36,8 @@ import { RESOURCES, TELEMETRY } from './lwm2m-profile-config.models'; -import {deepClone, isDefinedAndNotNull, isEqual, isUndefined} from '@core/utils'; -import {MatDialog} from '@angular/material/dialog'; -import {TranslateService} from '@ngx-translate/core'; +import { deepClone, isDefinedAndNotNull, isEqual, isUndefined } from '@core/utils'; +import { MatDialog } from '@angular/material/dialog'; import { Lwm2mObjectAddInstancesData, Lwm2mObjectAddInstancesDialogComponent @@ -83,10 +80,8 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor @Input() disabled: boolean; - constructor(private store: Store, - private fb: FormBuilder, - private dialog: MatDialog, - public translate: TranslateService) { + constructor(private fb: FormBuilder, + private dialog: MatDialog) { this.observeAttrTelemetryFormGroup = this.fb.group({ [CLIENT_LWM2M]: this.fb.array([]) }); @@ -98,7 +93,7 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor } private propagateChange = (v: any) => { - }; + } registerOnChange(fn: any): void { this.propagateChange = fn; @@ -189,7 +184,7 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor this.observeAttrTelemetryFormGroup.get(CLIENT_LWM2M).updateValueAndValidity(); } - trackByParams = (index: number, element: any): number => { + trackByParams = (index: number): number => { return index; } @@ -252,7 +247,6 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor } private updateInstancesIds = (data: Lwm2mObjectAddInstancesData): void => { - debugger const objectLwM2MFormGroup = (this.observeAttrTelemetryFormGroup.get(CLIENT_LWM2M) as FormArray).controls .find(e => e.value.keyId === data.objectKeyId) as FormGroup; const instancesArray = objectLwM2MFormGroup.value.instances as Instance []; @@ -318,7 +312,7 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor return objectName + ' <' + idVerObj + '>'; } getNameInstanceLwm2m = (instance: Instance, idVerObj: string): string => { - return ' instance <' + idVerObj + '/' + instance.id +'>'; + return ` instance <${idVerObj}/${instance.id}>`; } updateAttributeLwm2mObject = (event: Event, objectKeyId: number): void => { diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-components.module.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-components.module.ts index 43c297df56..4575d2bfff 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-components.module.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-components.module.ts @@ -14,19 +14,19 @@ /// limitations under the License. /// -import {NgModule} from '@angular/core'; -import {Lwm2mDeviceProfileTransportConfigurationComponent} from './lwm2m-device-profile-transport-configuration.component'; -import {Lwm2mObjectListComponent} from './lwm2m-object-list.component'; -import {Lwm2mObserveAttrTelemetryComponent} from './lwm2m-observe-attr-telemetry.component'; -import {Lwm2mObserveAttrTelemetryResourceComponent} from './lwm2m-observe-attr-telemetry-resource.component'; -import {Lwm2mAttributesDialogComponent} from './lwm2m-attributes-dialog.component'; -import {Lwm2mAttributesComponent} from './lwm2m-attributes.component'; -import {Lwm2mAttributesKeyListComponent} from './lwm2m-attributes-key-list.component'; -import {Lwm2mDeviceConfigServerComponent} from './lwm2m-device-config-server.component'; -import {Lwm2mObjectAddInstancesDialogComponent} from './lwm2m-object-add-instances-dialog.component'; -import {Lwm2mObjectAddInstancesListComponent} from './lwm2m-object-add-instances-list.component'; -import {CommonModule} from '@angular/common'; -import {SharedModule} from '@app/shared/shared.module'; +import { NgModule } from '@angular/core'; +import { Lwm2mDeviceProfileTransportConfigurationComponent } from './lwm2m-device-profile-transport-configuration.component'; +import { Lwm2mObjectListComponent } from './lwm2m-object-list.component'; +import { Lwm2mObserveAttrTelemetryComponent } from './lwm2m-observe-attr-telemetry.component'; +import { Lwm2mObserveAttrTelemetryResourceComponent } from './lwm2m-observe-attr-telemetry-resource.component'; +import { Lwm2mAttributesDialogComponent } from './lwm2m-attributes-dialog.component'; +import { Lwm2mAttributesComponent } from './lwm2m-attributes.component'; +import { Lwm2mAttributesKeyListComponent } from './lwm2m-attributes-key-list.component'; +import { Lwm2mDeviceConfigServerComponent } from './lwm2m-device-config-server.component'; +import { Lwm2mObjectAddInstancesDialogComponent } from './lwm2m-object-add-instances-dialog.component'; +import { Lwm2mObjectAddInstancesListComponent } from './lwm2m-object-add-instances-list.component'; +import { CommonModule } from '@angular/common'; +import { SharedModule } from '@app/shared/shared.module'; @NgModule({ declarations: diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts index ff927bc09f..f6e58df903 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-profile-config.models.ts @@ -127,8 +127,8 @@ export const SECURITY_CONFIG_MODE_NAMES = new Map( ); export interface ModelValue { - objectIds: string[], - objectsList: ObjectLwM2M[] + objectIds: string[]; + objectsList: ObjectLwM2M[]; } export interface BootstrapServersSecurityConfig { diff --git a/ui-ngx/src/app/modules/home/pages/device/device.module.ts b/ui-ngx/src/app/modules/home/pages/device/device.module.ts index f9ece52b22..8aeda0e2c1 100644 --- a/ui-ngx/src/app/modules/home/pages/device/device.module.ts +++ b/ui-ngx/src/app/modules/home/pages/device/device.module.ts @@ -24,8 +24,6 @@ import { DeviceCredentialsDialogComponent } from '@modules/home/pages/device/dev import { HomeDialogsModule } from '../../dialogs/home-dialogs.module'; import { HomeComponentsModule } from '@modules/home/components/home-components.module'; import { DeviceTabsComponent } from '@home/pages/device/device-tabs.component'; -import { SecurityConfigComponent } from '@home/pages/device/lwm2m/security-config.component'; -import { SecurityConfigServerComponent } from '@home/pages/device/lwm2m/security-config-server.component'; import { DefaultDeviceConfigurationComponent } from './data/default-device-configuration.component'; import { DeviceConfigurationComponent } from './data/device-configuration.component'; import { DeviceDataComponent } from './data/device-data.component'; @@ -50,9 +48,7 @@ import { SnmpDeviceTransportConfigurationComponent } from './data/snmp-device-tr DeviceComponent, DeviceTabsComponent, DeviceTableHeaderComponent, - DeviceCredentialsDialogComponent, - SecurityConfigComponent, - SecurityConfigServerComponent + DeviceCredentialsDialogComponent ], imports: [ CommonModule, diff --git a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.html b/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.html deleted file mode 100644 index bd02c9dee3..0000000000 --- a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.html +++ /dev/null @@ -1,101 +0,0 @@ - -
-
-
-
- - device.lwm2m-security-config.mode - - - {{ credentialTypeLwM2MNamesMap.get(securityConfigLwM2MType[securityMode]) }} - - - -
-
-
-
- - {{ 'device.lwm2m-security-config.client-publicKey-or-id' | translate }} - - {{clientPublicKeyOrId.value?.length || 0}}/{{lenMaxClientPublicKeyOrId}} - - {{ 'device.lwm2m-security-config.client-publicKey-or-id' | translate }} - {{ 'device.lwm2m-security-config.required' | translate }} - - - {{ 'device.lwm2m-security-config.client-key' | translate }} - {{ 'device.lwm2m-security-config.pattern_hex_dec' | translate: { - count: 0 - } }} - - - {{ 'device.lwm2m-security-config.client-key' | translate }} - {{ 'device.lwm2m-security-config.pattern_hex_dec' | translate: { - count: lenMaxClientPublicKeyOrId - } }} - - - - {{ 'device.lwm2m-security-config.client-secret-key' | translate }} - - {{clientSecretKey.value?.length || 0}}/{{lenMaxClientSecretKey}} - - {{ 'device.lwm2m-security-config.client-secret-key' | translate }} - {{ 'device.lwm2m-security-config.required' | translate }} - - - {{ 'device.lwm2m-security-config.client-key' | translate }} - {{ 'device.lwm2m-security-config.pattern_hex_dec' | translate: { - count: 0 - } }} - - - {{ 'device.lwm2m-security-config.client-key' | translate }} - {{ 'device.lwm2m-security-config.pattern_hex_dec' | translate: { - count: lenMaxClientSecretKey - } }} - - -
-
-
-
diff --git a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.ts b/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.ts deleted file mode 100644 index a068bb589e..0000000000 --- a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config-server.component.ts +++ /dev/null @@ -1,142 +0,0 @@ -/// -/// Copyright © 2016-2021 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - -import { Component, forwardRef, Inject, Input, OnInit } from '@angular/core'; -import { ControlValueAccessor, FormBuilder, FormGroup, NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; -import { - DeviceCredentialsDialogLwm2mData, - KEY_REGEXP_HEX_DEC, - LEN_MAX_PRIVATE_KEY, - LEN_MAX_PSK, - LEN_MAX_PUBLIC_KEY_RPK, - LEN_MAX_PUBLIC_KEY_X509, - SECURITY_CONFIG_MODE, - SECURITY_CONFIG_MODE_NAMES, - ServerSecurityConfig -} from '@home/pages/device/lwm2m/security-config.models'; -import { Store } from '@ngrx/store'; -import { AppState } from '@core/core.state'; -import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog'; -import { PageComponent } from '@shared/components/page.component'; - -@Component({ - selector: 'tb-security-config-server-lwm2m', - templateUrl: './security-config-server.component.html', - styleUrls: [], - providers: [ - { - provide: NG_VALUE_ACCESSOR, - useExisting: forwardRef(() => SecurityConfigServerComponent), - multi: true - } - ] -}) - -export class SecurityConfigServerComponent extends PageComponent implements OnInit, ControlValueAccessor { - - securityConfigLwM2MType = SECURITY_CONFIG_MODE; - securityConfigLwM2MTypes = Object.keys(SECURITY_CONFIG_MODE); - credentialTypeLwM2MNamesMap = SECURITY_CONFIG_MODE_NAMES; - lenMinClientPublicKeyOrId = 0; - lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; - lenMinClientSecretKey = LEN_MAX_PRIVATE_KEY; - lenMaxClientSecretKey = LEN_MAX_PRIVATE_KEY; - - @Input() serverFormGroup: FormGroup; - - constructor(protected store: Store, - @Inject(MAT_DIALOG_DATA) public data: DeviceCredentialsDialogLwm2mData, - public dialogRef: MatDialogRef, - public fb: FormBuilder) { - super(store); - } - - ngOnInit(): void { - this.registerDisableOnLoadFormControl(this.serverFormGroup.get('securityMode')); - } - - private updateValueFields(serverData: ServerSecurityConfig): void { - this.serverFormGroup.patchValue(serverData, {emitEvent: false}); - const securityMode = this.serverFormGroup.get('securityMode').value as SECURITY_CONFIG_MODE; - this.updateValidate(securityMode); - } - - private updateValidate(securityMode: SECURITY_CONFIG_MODE): void { - switch (securityMode) { - case SECURITY_CONFIG_MODE.NO_SEC: - this.serverFormGroup.get('clientPublicKeyOrId').setValidators([]); - this.serverFormGroup.get('clientSecretKey').setValidators([]); - break; - case SECURITY_CONFIG_MODE.PSK: - this.lenMinClientPublicKeyOrId = 0; - this.lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; - this.lenMinClientSecretKey = LEN_MAX_PSK; - this.lenMaxClientSecretKey = LEN_MAX_PSK; - this.setValidatorsSecurity(securityMode); - break; - case SECURITY_CONFIG_MODE.RPK: - this.lenMinClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; - this.lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_RPK; - this.lenMinClientSecretKey = LEN_MAX_PRIVATE_KEY; - this.lenMaxClientSecretKey = LEN_MAX_PRIVATE_KEY; - this.setValidatorsSecurity(securityMode); - break; - case SECURITY_CONFIG_MODE.X509: - this.lenMinClientPublicKeyOrId = 0; - this.lenMaxClientPublicKeyOrId = LEN_MAX_PUBLIC_KEY_X509; - this.lenMinClientSecretKey = LEN_MAX_PRIVATE_KEY; - this.lenMaxClientSecretKey = LEN_MAX_PRIVATE_KEY; - this.setValidatorsSecurity(securityMode); - break; - } - this.serverFormGroup.updateValueAndValidity(); - } - - private setValidatorsSecurity = (securityMode: SECURITY_CONFIG_MODE): void => { - if (securityMode === SECURITY_CONFIG_MODE.PSK) { - this.serverFormGroup.get('clientPublicKeyOrId').setValidators([Validators.required]); - } else { - this.serverFormGroup.get('clientPublicKeyOrId').setValidators([Validators.required, - Validators.pattern(KEY_REGEXP_HEX_DEC), - Validators.minLength(this.lenMinClientPublicKeyOrId), - Validators.maxLength(this.lenMaxClientPublicKeyOrId)]); - } - - this.serverFormGroup.get('clientSecretKey').setValidators([Validators.required, - Validators.pattern(KEY_REGEXP_HEX_DEC), - Validators.minLength(this.lenMinClientSecretKey), - Validators.maxLength(this.lenMaxClientSecretKey)]); - } - - securityModeChanged(securityMode: SECURITY_CONFIG_MODE): void { - this.updateValidate(securityMode); - } - - writeValue(value: any): void { - if (value) { - this.updateValueFields(value); - } - } - - registerOnChange(fn: (value: any) => any): void { - } - - registerOnTouched(fn: any): void { - } - - setDisabledState?(isDisabled: boolean): void { - } -} diff --git a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.html b/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.html deleted file mode 100644 index db0946b371..0000000000 --- a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.html +++ /dev/null @@ -1,180 +0,0 @@ - -
- -

{{ title }}

- - - -
-
- - device.lwm2m-security-config.endpoint - - - {{ 'device.lwm2m-security-config.endpoint' | translate }} - {{ 'device.lwm2m-security-config.required' | translate }} - - - - - -
- - device.lwm2m-security-config.mode - - - {{ credentialTypeLwM2MNamesMap.get(securityConfigLwM2MType[securityConfigClientMode]) }} - - - -
- - {{ 'device.lwm2m-security-config.identity' | translate }} - - - {{ 'device.lwm2m-security-config.identity' | translate }} - {{ 'device.lwm2m-security-config.required' | translate }} - - -
-
- - {{ 'device.lwm2m-security-config.client-key' | translate }} - - {{clientKey.value?.length || 0}}/{{lenMaxKeyClient}} - - {{ 'device.lwm2m-security-config.client-key' | translate }} - {{ 'device.lwm2m-security-config.required' | translate }} - - - {{ 'device.lwm2m-security-config.client-key' | translate }} - {{ 'device.lwm2m-security-config.pattern_hex_dec' | translate: { - count: 0 - } }} - - - {{ 'device.lwm2m-security-config.client-key' | translate }} - {{ 'device.lwm2m-security-config.pattern_hex_dec' | translate: { - count: lenMaxKeyClient - } }} - - -
-
- - {{ 'device.lwm2m-security-config.client-certificate' | translate }} - -
-
- - - - -
- - - - -
{{ 'device.lwm2m-security-config.bootstrap-server' | translate | uppercase }}
- - - -
- - -
- - - - - - - -
{{ 'device.lwm2m-security-config.lwm2m-server' | translate | uppercase }}
- - - -
- - -
- - - -
- - - - -
-
- - -
-
- - - -
-
-
- - - -
-
diff --git a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.ts b/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.ts deleted file mode 100644 index 028f28c72a..0000000000 --- a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.component.ts +++ /dev/null @@ -1,382 +0,0 @@ -/// -/// Copyright © 2016-2021 The Thingsboard Authors -/// -/// Licensed under the Apache License, Version 2.0 (the "License"); -/// you may not use this file except in compliance with the License. -/// You may obtain a copy of the License at -/// -/// http://www.apache.org/licenses/LICENSE-2.0 -/// -/// Unless required by applicable law or agreed to in writing, software -/// distributed under the License is distributed on an "AS IS" BASIS, -/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -/// See the License for the specific language governing permissions and -/// limitations under the License. -/// - - -import {Component, Inject, OnInit} from '@angular/core'; -import {DialogComponent} from '@shared/components/dialog.component'; -import {Store} from '@ngrx/store'; -import {AppState} from '@core/core.state'; -import {Router} from '@angular/router'; -import {MAT_DIALOG_DATA, MatDialogRef} from '@angular/material/dialog'; -import {FormBuilder, FormGroup, Validators} from '@angular/forms'; -import {TranslateService} from '@ngx-translate/core'; -import { - BOOTSTRAP_SERVER, - BOOTSTRAP_SERVERS, - ClientSecurityConfig, - DeviceCredentialsDialogLwm2mData, - getClientSecurityConfig, - JSON_ALL_CONFIG, - KEY_REGEXP_HEX_DEC, - LEN_MAX_PSK, - LEN_MAX_PUBLIC_KEY_RPK, - LWM2M_SERVER, - SECURITY_CONFIG_MODE, - SECURITY_CONFIG_MODE_NAMES, - SecurityConfigModels -} from './security-config.models'; -import {WINDOW} from '@core/services/window.service'; -import {MatTabChangeEvent} from '@angular/material/tabs'; -import {MatTab} from '@angular/material/tabs/tab'; - -@Component({ - selector: 'tb-security-config-lwm2m', - templateUrl: './security-config.component.html', - styleUrls: [] -}) - -export class SecurityConfigComponent extends DialogComponent implements OnInit { - - lwm2mConfigFormGroup: FormGroup; - title: string; - submitted = false; - securityConfigLwM2MType = SECURITY_CONFIG_MODE; - securityConfigLwM2MTypes = Object.keys(SECURITY_CONFIG_MODE); - credentialTypeLwM2MNamesMap = SECURITY_CONFIG_MODE_NAMES; - formControlNameJsonAllConfig = JSON_ALL_CONFIG; - jsonAllConfig: SecurityConfigModels; - bootstrapServers: string; - bootstrapServer: string; - lwm2mServer: string; - jsonObserveData: {}; - lenMaxKeyClient = LEN_MAX_PSK; - tabPrevious: MatTab; - tabIndexPrevious = 0; - - constructor(protected store: Store, - protected router: Router, - @Inject(MAT_DIALOG_DATA) public data: DeviceCredentialsDialogLwm2mData, - public dialogRef: MatDialogRef, - public fb: FormBuilder, - public translate: TranslateService, - @Inject(WINDOW) private window: Window) { - super(store, router, dialogRef); - } - - ngOnInit(): void { - this.jsonAllConfig = JSON.parse(JSON.stringify(this.data.jsonAllConfig)) as SecurityConfigModels; - this.initConstants(); - this.lwm2mConfigFormGroup = this.initLwm2mConfigFormGroup(); - this.title = this.translate.instant('device.lwm2m-security-info') + ': ' + this.data.endPoint; - this.lwm2mConfigFormGroup.get('clientCertificate').disable(); - this.initClientSecurityConfig(this.lwm2mConfigFormGroup.get('jsonAllConfig').value); - this.registerDisableOnLoadFormControl(this.lwm2mConfigFormGroup.get('securityConfigClientMode')); - } - - private initConstants = (): void => { - this.bootstrapServers = BOOTSTRAP_SERVERS; - this.bootstrapServer = BOOTSTRAP_SERVER; - this.lwm2mServer = LWM2M_SERVER; - } - - /** - * initChildesFormGroup - */ - get bootstrapFormGroup(): FormGroup { - return this.lwm2mConfigFormGroup.get('bootstrapFormGroup') as FormGroup; - } - - get lwm2mServerFormGroup(): FormGroup { - return this.lwm2mConfigFormGroup.get('lwm2mServerFormGroup') as FormGroup; - } - - get observeAttrFormGroup(): FormGroup { - return this.lwm2mConfigFormGroup.get('observeFormGroup') as FormGroup; - } - - private initClientSecurityConfig = (jsonAllConfig: SecurityConfigModels): void => { - switch (jsonAllConfig.client.securityConfigClientMode) { - case SECURITY_CONFIG_MODE.NO_SEC: - break; - case SECURITY_CONFIG_MODE.PSK: - const clientSecurityConfigPSK = jsonAllConfig.client as ClientSecurityConfig; - this.lwm2mConfigFormGroup.patchValue({ - identityPSK: clientSecurityConfigPSK.identity, - clientKey: clientSecurityConfigPSK.key, - }, {emitEvent: false}); - break; - case SECURITY_CONFIG_MODE.RPK: - const clientSecurityConfigRPK = jsonAllConfig.client as ClientSecurityConfig; - this.lwm2mConfigFormGroup.patchValue({ - clientKey: clientSecurityConfigRPK.key, - }, {emitEvent: false}); - break; - case SECURITY_CONFIG_MODE.X509: - const clientSecurityConfigX509 = jsonAllConfig.client as ClientSecurityConfig; - this.lwm2mConfigFormGroup.patchValue({ - clientCertificate: clientSecurityConfigX509.x509 - }, {emitEvent: false}); - break; - } - this.securityConfigClientUpdateValidators(this.lwm2mConfigFormGroup.get('securityConfigClientMode').value); - } - - securityConfigClientModeChanged = (mode: SECURITY_CONFIG_MODE): void => { - switch (mode) { - case SECURITY_CONFIG_MODE.NO_SEC: - const clientSecurityConfigNoSEC = getClientSecurityConfig(mode) as ClientSecurityConfig; - this.jsonAllConfig.client = clientSecurityConfigNoSEC; - this.lwm2mConfigFormGroup.patchValue({ - jsonAllConfig: this.jsonAllConfig, - clientCertificate: false - }, {emitEvent: false}); - break; - case SECURITY_CONFIG_MODE.PSK: - const clientSecurityConfigPSK = getClientSecurityConfig(mode, this.lwm2mConfigFormGroup.get('endPoint') - .value) as ClientSecurityConfig; - clientSecurityConfigPSK.identity = this.data.endPoint; - clientSecurityConfigPSK.key = this.lwm2mConfigFormGroup.get('clientKey').value; - this.jsonAllConfig.client = clientSecurityConfigPSK; - this.lwm2mConfigFormGroup.patchValue({ - identityPSK: clientSecurityConfigPSK.identity, - clientCertificate: false - }, {emitEvent: false}); - break; - case SECURITY_CONFIG_MODE.RPK: - const clientSecurityConfigRPK = getClientSecurityConfig(mode) as ClientSecurityConfig; - clientSecurityConfigRPK.key = this.lwm2mConfigFormGroup.get('clientKey').value; - this.jsonAllConfig.client = clientSecurityConfigRPK; - this.lwm2mConfigFormGroup.patchValue({ - clientCertificate: false - }, {emitEvent: false}); - break; - case SECURITY_CONFIG_MODE.X509: - this.jsonAllConfig.client = getClientSecurityConfig(mode) as ClientSecurityConfig; - this.lwm2mConfigFormGroup.patchValue({ - clientCertificate: true - }, {emitEvent: false}); - break; - } - this.securityConfigClientUpdateValidators(mode); - } - - private securityConfigClientUpdateValidators = (mode: SECURITY_CONFIG_MODE): void => { - switch (mode) { - case SECURITY_CONFIG_MODE.NO_SEC: - this.setValidatorsNoSecX509(); - break; - case SECURITY_CONFIG_MODE.PSK: - this.lenMaxKeyClient = LEN_MAX_PSK; - this.setValidatorsPskRpk(mode); - break; - case SECURITY_CONFIG_MODE.RPK: - this.lenMaxKeyClient = LEN_MAX_PUBLIC_KEY_RPK; - this.setValidatorsPskRpk(mode); - break; - case SECURITY_CONFIG_MODE.X509: - this.lenMaxKeyClient = LEN_MAX_PUBLIC_KEY_RPK; - this.setValidatorsNoSecX509(); - break; - } - this.lwm2mConfigFormGroup.updateValueAndValidity(); - } - - private setValidatorsNoSecX509 = (): void => { - this.lwm2mConfigFormGroup.get('identityPSK').setValidators([]); - this.lwm2mConfigFormGroup.get('clientKey').setValidators([]); - } - - private setValidatorsPskRpk = (mode: SECURITY_CONFIG_MODE): void => { - if (mode === SECURITY_CONFIG_MODE.PSK) { - this.lwm2mConfigFormGroup.get('identityPSK').setValidators([Validators.required]); - } else { - this.lwm2mConfigFormGroup.get('identityPSK').setValidators([]); - } - this.lwm2mConfigFormGroup.get('clientKey').setValidators([Validators.required, - Validators.pattern(KEY_REGEXP_HEX_DEC), - Validators.maxLength(this.lenMaxKeyClient), Validators.minLength(this.lenMaxKeyClient)]); - } - - tabChanged = (tabChangeEvent: MatTabChangeEvent): void => { - if (this.tabIndexPrevious !== tabChangeEvent.index) { this.upDateValueToJson(); } - this.tabIndexPrevious = tabChangeEvent.index; - } - - private upDateValueToJson(): void { - switch (this.tabIndexPrevious) { - case 0: - this.upDateValueToJsonTab0(); - break; - case 1: - this.upDateValueToJsonTab1(); - break; - case 2: - this.upDateValueToJsonTab2(); - break; - } - } - - private upDateValueToJsonTab0 = (): void => { - if (this.lwm2mConfigFormGroup !== null) { - if (!this.lwm2mConfigFormGroup.get('endPoint').pristine && this.lwm2mConfigFormGroup.get('endPoint').valid) { - this.data.endPoint = this.lwm2mConfigFormGroup.get('endPoint').value; - /** Client mode == PSK */ - if (this.lwm2mConfigFormGroup.get('securityConfigClientMode').value === SECURITY_CONFIG_MODE.PSK) { - const endPoint = 'endpoint'; - this.jsonAllConfig.client[endPoint] = this.data.endPoint; - this.lwm2mConfigFormGroup.get('endPoint').markAsPristine({ - onlySelf: true - }); - this.upDateJsonAllConfig(); - } - } - /** only Client mode == PSK */ - if (!this.lwm2mConfigFormGroup.get('identityPSK').pristine && this.lwm2mConfigFormGroup.get('identityPSK').valid) { - this.lwm2mConfigFormGroup.get('identityPSK').markAsPristine({ - onlySelf: true - }); - this.updateIdentityPSK(); - } - /** only Client mode == PSK (len = 64) || RPK (len = 182) */ - if (!this.lwm2mConfigFormGroup.get('clientKey').pristine && this.lwm2mConfigFormGroup.get('clientKey').valid) { - this.lwm2mConfigFormGroup.get('clientKey').markAsPristine({ - onlySelf: true - }); - this.updateClientKey(); - } - } - } - - private upDateValueToJsonTab1 = (): void => { - if (this.lwm2mConfigFormGroup !== null) { - if (this.bootstrapFormGroup !== null && !this.bootstrapFormGroup.pristine && this.bootstrapFormGroup.valid) { - this.jsonAllConfig.bootstrap.bootstrapServer = this.bootstrapFormGroup.value; - this.bootstrapFormGroup.markAsPristine({ - onlySelf: true - }); - this.upDateJsonAllConfig(); - } - - if (this.lwm2mServerFormGroup !== null && !this.lwm2mServerFormGroup.pristine && this.lwm2mServerFormGroup.valid) { - this.jsonAllConfig.bootstrap.lwm2mServer = this.lwm2mServerFormGroup.value; - this.lwm2mServerFormGroup.markAsPristine({ - onlySelf: true - }); - this.upDateJsonAllConfig(); - } - } - } - - private upDateValueToJsonTab2 = (): void => { - if (!this.lwm2mConfigFormGroup.get(this.formControlNameJsonAllConfig).pristine && - this.lwm2mConfigFormGroup.get(this.formControlNameJsonAllConfig).valid) { - this.jsonAllConfig = this.lwm2mConfigFormGroup.get(this.formControlNameJsonAllConfig).value; - this.lwm2mConfigFormGroup.get(this.formControlNameJsonAllConfig).markAsPristine({ - onlySelf: true - }); - } - } - - private updateIdentityPSK = (): void => { - const securityMode = 'securityMode'; - if (this.lwm2mConfigFormGroup.get('bootstrapServer').value[securityMode] === SECURITY_CONFIG_MODE.PSK) { - this.lwm2mConfigFormGroup.get('bootstrapFormGroup').patchValue({ - clientPublicKeyOrId: this.lwm2mConfigFormGroup.get('identityPSK').value - }); - const identity = 'identity'; - this.jsonAllConfig.client[identity] = this.lwm2mConfigFormGroup.get('identityPSK').value; - this.upDateJsonAllConfig(); - } - if (this.lwm2mConfigFormGroup.get('lwm2mServer').value[securityMode] === SECURITY_CONFIG_MODE.PSK) { - this.lwm2mConfigFormGroup.get('lwm2mServerFormGroup').patchValue({ - clientPublicKeyOrId: this.lwm2mConfigFormGroup.get('identityPSK').value - }); - this.jsonAllConfig.bootstrap.lwm2mServer.clientPublicKeyOrId = this.lwm2mConfigFormGroup.get('identityPSK').value; - this.upDateJsonAllConfig(); - } - } - - private updateClientKey = (): void => { - const key = 'key'; - const securityMode = 'securityMode'; - this.jsonAllConfig.client[key] = this.lwm2mConfigFormGroup.get('clientKey').value; - if (this.lwm2mConfigFormGroup.get('bootstrapServer').value[securityMode] === SECURITY_CONFIG_MODE.PSK) { - this.lwm2mConfigFormGroup.get('bootstrapServer').patchValue({ - clientSecretKey: this.jsonAllConfig.client[key] - }, {emitEvent: false}); - this.jsonAllConfig.bootstrap.bootstrapServer.clientSecretKey = this.jsonAllConfig.client[key]; - } - if (this.lwm2mConfigFormGroup.get('lwm2mServer').value[securityMode] === SECURITY_CONFIG_MODE.PSK) { - this.lwm2mConfigFormGroup.get('lwm2mServer').patchValue({ - clientSecretKey: this.jsonAllConfig.client[key] - }, {emitEvent: false}); - this.jsonAllConfig.bootstrap.lwm2mServer.clientSecretKey = this.jsonAllConfig.client[key]; - } - this.upDateJsonAllConfig(); - } - - private upDateJsonAllConfig = (): void => { - this.data.jsonAllConfig = JSON.parse(JSON.stringify(this.jsonAllConfig)); - this.lwm2mConfigFormGroup.patchValue({ - jsonAllConfig: JSON.parse(JSON.stringify(this.jsonAllConfig)) - }, {emitEvent: false}); - this.lwm2mConfigFormGroup.markAsDirty(); - } - - private initLwm2mConfigFormGroup = (): FormGroup => { - if (SECURITY_CONFIG_MODE[this.jsonAllConfig.client.securityConfigClientMode.toString()] === SECURITY_CONFIG_MODE.PSK) { - const endpoint = 'endpoint'; - this.data.endPoint = this.jsonAllConfig.client[endpoint]; - } - return this.fb.group({ - securityConfigClientMode: [SECURITY_CONFIG_MODE[this.jsonAllConfig.client.securityConfigClientMode.toString()], []], - identityPSK: ['', []], - clientKey: ['', []], - clientCertificate: [false, []], - bootstrapServer: [this.jsonAllConfig.bootstrap[this.bootstrapServer], []], - lwm2mServer: [this.jsonAllConfig.bootstrap[this.lwm2mServer], []], - bootstrapFormGroup: this.getServerGroup(), - lwm2mServerFormGroup: this.getServerGroup(), - endPoint: [this.data.endPoint, []], - jsonAllConfig: [this.jsonAllConfig, []] - }); - } - - private getServerGroup = (): FormGroup => { - return this.fb.group({ - securityMode: [this.fb.control(SECURITY_CONFIG_MODE.NO_SEC), []], - clientPublicKeyOrId: ['', []], - clientSecretKey: ['', []] - }); - } - - save(): void { - this.upDateValueToJson(); - this.data.endPoint = this.lwm2mConfigFormGroup.get('endPoint').value.split('\'').join(''); - this.data.jsonAllConfig = this.jsonAllConfig; - if (this.lwm2mConfigFormGroup.get('securityConfigClientMode').value === SECURITY_CONFIG_MODE.PSK) { - const identity = 'identity'; - this.data.endPoint = this.data.jsonAllConfig.client[identity]; - } - this.dialogRef.close(this.data); - } - - cancel(): void { - this.dialogRef.close(undefined); - } -} - - diff --git a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.models.ts b/ui-ngx/src/app/shared/models/lwm2m-security-config.models.ts similarity index 60% rename from ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.models.ts rename to ui-ngx/src/app/shared/models/lwm2m-security-config.models.ts index 32b8026a4c..de0f2dc8a5 100644 --- a/ui-ngx/src/app/modules/home/pages/device/lwm2m/security-config.models.ts +++ b/ui-ngx/src/app/shared/models/lwm2m-security-config.models.ts @@ -17,9 +17,6 @@ export const JSON_ALL_CONFIG = 'jsonAllConfig'; export const END_POINT = 'endPoint'; export const DEFAULT_END_POINT = 'default_client_lwm2m_end_point_no_sec'; -export const BOOTSTRAP_SERVERS = 'servers'; -export const BOOTSTRAP_SERVER = 'bootstrapServer'; -export const LWM2M_SERVER = 'lwm2mServer'; export const LEN_MAX_PSK = 64; export const LEN_MAX_PRIVATE_KEY = 134; export const LEN_MAX_PUBLIC_KEY_RPK = 182; @@ -28,28 +25,28 @@ export const KEY_REGEXP_HEX_DEC = /^[-+]?[0-9A-Fa-f]+\.?[0-9A-Fa-f]*?$/; export interface DeviceCredentialsDialogLwm2mData { - jsonAllConfig?: SecurityConfigModels; + jsonAllConfig?: Lwm2mSecurityConfigModels; endPoint?: string; } -export enum SECURITY_CONFIG_MODE { +export enum Lwm2mSecurityType { PSK = 'PSK', RPK = 'RPK', X509 = 'X509', NO_SEC = 'NO_SEC' } -export const SECURITY_CONFIG_MODE_NAMES = new Map( +export const Lwm2mSecurityTypeTranslationMap = new Map( [ - [SECURITY_CONFIG_MODE.PSK, 'Pre-Shared Key'], - [SECURITY_CONFIG_MODE.RPK, 'Raw Public Key'], - [SECURITY_CONFIG_MODE.X509, 'X.509 Certificate'], - [SECURITY_CONFIG_MODE.NO_SEC, 'No Security'], + [Lwm2mSecurityType.PSK, 'Pre-Shared Key'], + [Lwm2mSecurityType.RPK, 'Raw Public Key'], + [Lwm2mSecurityType.X509, 'X.509 Certificate'], + [Lwm2mSecurityType.NO_SEC, 'No Security'], ] ); export interface ClientSecurityConfig { - securityConfigClientMode: string; + securityConfigClientMode: Lwm2mSecurityType; endpoint: string; identity: string; key: string; @@ -57,7 +54,7 @@ export interface ClientSecurityConfig { } export interface ServerSecurityConfig { - securityMode: string; + securityMode: Lwm2mSecurityType; clientPublicKeyOrId?: string; clientSecretKey?: string; } @@ -67,20 +64,19 @@ interface BootstrapSecurityConfig { lwm2mServer: ServerSecurityConfig; } -export interface SecurityConfigModels { +export interface Lwm2mSecurityConfigModels { client: ClientSecurityConfig; bootstrap: BootstrapSecurityConfig; } -export function getClientSecurityConfig(securityConfigMode: SECURITY_CONFIG_MODE, endPoint?: string): ClientSecurityConfig { - let security = getDefaultClientSecurityConfig(); - security.securityConfigClientMode = securityConfigMode.toString(); +export function getClientSecurityConfig(securityConfigMode: Lwm2mSecurityType, endPoint = ''): ClientSecurityConfig { + const security = getDefaultClientSecurityConfig(securityConfigMode); switch (securityConfigMode) { - case SECURITY_CONFIG_MODE.PSK: + case Lwm2mSecurityType.PSK: security.endpoint = endPoint; security.identity = endPoint; break; - case SECURITY_CONFIG_MODE.X509: + case Lwm2mSecurityType.X509: security.x509 = true; break; } @@ -88,9 +84,9 @@ export function getClientSecurityConfig(securityConfigMode: SECURITY_CONFIG_MODE return security; } -export function getDefaultClientSecurityConfig(): ClientSecurityConfig { +export function getDefaultClientSecurityConfig(securityConfigMode: Lwm2mSecurityType): ClientSecurityConfig { return { - securityConfigClientMode: SECURITY_CONFIG_MODE.NO_SEC.toString(), + securityConfigClientMode: securityConfigMode, endpoint: '', identity: '', key: '', @@ -100,7 +96,7 @@ export function getDefaultClientSecurityConfig(): ClientSecurityConfig { export function getDefaultServerSecurityConfig(): ServerSecurityConfig { return { - securityMode: SECURITY_CONFIG_MODE.NO_SEC.toString(), + securityMode: Lwm2mSecurityType.NO_SEC, clientPublicKeyOrId: '', clientSecretKey: '' }; @@ -113,46 +109,42 @@ function getDefaultBootstrapSecurityConfig(): BootstrapSecurityConfig { }; } -export function getDefaultSecurityConfig(): SecurityConfigModels { +export function getDefaultSecurityConfig(): Lwm2mSecurityConfigModels { const securityConfigModels = { - client: getClientSecurityConfig(SECURITY_CONFIG_MODE.NO_SEC), + client: getClientSecurityConfig(Lwm2mSecurityType.NO_SEC), bootstrap: getDefaultBootstrapSecurityConfig() }; return securityConfigModels; } -const isSecurityConfigModels = (p: any): p is SecurityConfigModels => +const isSecurityConfigModels = (p: any): boolean => p.hasOwnProperty('client') && - isClientSecurityConfigType(p['client']) && + isClientSecurityConfigType(p.client) && p.hasOwnProperty('bootstrap') && - isBootstrapSecurityConfig(p['bootstrap']); + isBootstrapSecurityConfig(p.bootstrap); -const isClientSecurityConfigType = (p: any): p is ClientSecurityConfig => +const isClientSecurityConfigType = (p: any): boolean => p.hasOwnProperty('securityConfigClientMode') && p.hasOwnProperty('endpoint') && p.hasOwnProperty('identity') && p.hasOwnProperty('key') && p.hasOwnProperty('x509'); -const isBootstrapSecurityConfig = (p: any): p is BootstrapSecurityConfig => +const isBootstrapSecurityConfig = (p: any): boolean => p.hasOwnProperty('bootstrapServer') && - isServerSecurityConfig(p['bootstrapServer']) && + isServerSecurityConfig(p.bootstrapServer) && p.hasOwnProperty('lwm2mServer') && - isServerSecurityConfig(p['lwm2mServer']); + isServerSecurityConfig(p.lwm2mServer); -const isServerSecurityConfig = (p: any): p is ServerSecurityConfig => +const isServerSecurityConfig = (p: any): boolean => p.hasOwnProperty('securityMode') && p.hasOwnProperty('clientPublicKeyOrId') && p.hasOwnProperty('clientSecretKey'); export function validateSecurityConfig(config: string): boolean { try { - const securityConfig= JSON.parse(config); - if (isSecurityConfigModels(securityConfig)) { - return true; - } else { - return false; - } + const securityConfig = JSON.parse(config); + return isSecurityConfigModels(securityConfig); } catch (e) { return false; } diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 08335a0175..bd7ec5e2a2 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -928,9 +928,13 @@ "lwm2m-credentials-value-tip": "Edit security config json editor", "lwm2m-security-config": { "identity": "Client Identity", + "identity-required": "Client Identity is required.", "client-key": "Client Key", - "required": " value is required.", + "client-key-required": "Client Key is required.", + "client-key-pattern": "Client Key must be hexadecimal format.", + "client-key-length": "Client Key must be {{ count }} characters.", "endpoint": "Endpoint Client Name", + "endpoint-required": "Endpoint Client Name is required.", "mode": "Security config mode", "client-tab": "Client Security Config", "client-certificate": "Client certificate", @@ -938,9 +942,14 @@ "bootstrap-server": "Bootstrap Server", "lwm2m-server": "LwM2M Server", "client-publicKey-or-id": "Client Public Key or Id", + "client-publicKey-or-id-required": "Client Public Key or Id is required.", + "client-publicKey-or-id-pattern": "Client Public Key or Id must be hexadecimal format.", + "client-publicKey-or-id-length": "Client Public Key or Id must be {{ count }} characters.", "client-secret-key": "Client Secret Key", - "config-json-tab": "Json Client Security Config", - "pattern_hex_dec": "{ count, plural, 0 {must be hex decimal format} other {must be # characters} }" + "client-secret-key-required": "Client Secret Key is required.", + "client-secret-key-pattern": "Client Secret Key must be hexadecimal format.", + "client-secret-key-length": "Client Secret Key must be {{ count }} characters.", + "config-json-tab": "Json Client Security Config" }, "client-id": "Client ID", "client-id-pattern": "Contains invalid character.", From e33420d3a89264b3df099654eea8b3671ca0fbec Mon Sep 17 00:00:00 2001 From: nickAS21 <44275303+nickAS21@users.noreply.github.com> Date: Fri, 30 Apr 2021 08:39:06 +0300 Subject: [PATCH 31/32] Lwm2m: firmwareUpdate (#4516) * Lwm2m: firmwareUpdate * Lwm2m: firmwareUpdate with merge master * Lwm2m: firmwareUpdate cleaned * Lwm2m: delete Californium.properties * Lwm2m: merge with master --- .../resource/DefaultTbResourceService.java | 2 +- .../server/common/data/DataConstants.java | 4 +- ...TransportBootstrapServerConfiguration.java | 3 +- .../lwm2m/server/LwM2mNetworkConfig.java | 100 +++++++ .../lwm2m/server/LwM2mServerListener.java | 14 +- .../lwm2m/server/LwM2mTransportHandler.java | 33 +-- .../lwm2m/server/LwM2mTransportRequest.java | 123 +++++---- .../LwM2mTransportServerConfiguration.java | 8 +- .../lwm2m/server/LwM2mTransportService.java | 4 +- .../server/LwM2mTransportServiceImpl.java | 255 ++++++++++++------ .../server/adaptors/LwM2MJsonAdaptor.java | 21 +- .../adaptors/LwM2MTransportAdaptor.java | 4 +- .../lwm2m/server/client/LwM2mClient.java | 18 +- .../server/client/LwM2mClientContextImpl.java | 10 +- .../server/client/LwM2mFirmwareUpdate.java | 27 ++ .../lwm2m-observe-attr-telemetry.component.ts | 15 +- .../assets/locale/locale.constant-en_US.json | 2 +- 17 files changed, 435 insertions(+), 208 deletions(-) create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java create mode 100644 common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFirmwareUpdate.java diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java index 81c59609ff..63c587405a 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java @@ -185,7 +185,7 @@ public class DefaultTbResourceService implements TbResourceService { instance.setId(0); List resources = new ArrayList<>(); obj.resources.forEach((k, v) -> { - if (!v.operations.isExecutable()) { + if (v.operations.isReadable()) { LwM2mResourceObserve lwM2MResourceObserve = new LwM2mResourceObserve(k, v.name, false, false, false); resources.add(lwM2MResourceObserve); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index d0fdf169fc..094069dafb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -95,8 +95,8 @@ public class DataConstants { //firmware //telemetry - public static final String CURRENT_FIRMWARE_TITLE = "cur_fw_title"; - public static final String CURRENT_FIRMWARE_VERSION = "cur_fw_version"; + public static final String CURRENT_FIRMWARE_TITLE = "current_fw_title"; + public static final String CURRENT_FIRMWARE_VERSION = "current_fw_version"; public static final String TARGET_FIRMWARE_TITLE = "target_fw_title"; public static final String TARGET_FIRMWARE_VERSION = "target_fw_version"; public static final String TARGET_FIRMWARE_TS = "target_fw_ts"; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/LwM2MTransportBootstrapServerConfiguration.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/LwM2MTransportBootstrapServerConfiguration.java index e531de3628..9e40c736df 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/LwM2MTransportBootstrapServerConfiguration.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/bootstrap/LwM2MTransportBootstrapServerConfiguration.java @@ -55,7 +55,7 @@ import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_ECDHE import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8; import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_PSK_WITH_AES_128_CBC_SHA256; import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_PSK_WITH_AES_128_CCM_8; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.getCoapConfig; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mNetworkConfig.getCoapConfig; @Slf4j @Component @@ -93,7 +93,6 @@ public class LwM2MTransportBootstrapServerConfiguration { builder.setCoapConfig(getCoapConfig(bootstrapPortNoSec, bootstrapSecurePort)); /** Define model provider (Create Models )*/ -// builder.setModel(new StaticModel(contextS.getLwM2MTransportConfigServer().getModelsValueCommon())); /** Create credentials */ this.setServerWithCredentials(builder); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java new file mode 100644 index 0000000000..22f822bae6 --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mNetworkConfig.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server; + +import org.eclipse.californium.core.network.config.NetworkConfig; + +public class LwM2mNetworkConfig { + + public static NetworkConfig getCoapConfig(Integer serverPortNoSec, Integer serverSecurePort) { + NetworkConfig coapConfig = new NetworkConfig(); + coapConfig.setInt(NetworkConfig.Keys.COAP_PORT,serverPortNoSec); + coapConfig.setInt(NetworkConfig.Keys.COAP_SECURE_PORT,serverSecurePort); + /** + * Example:Property for large packet: + * #NetworkConfig config = new NetworkConfig(); + * #config.setInt(NetworkConfig.Keys.MAX_MESSAGE_SIZE,32); + * #config.setInt(NetworkConfig.Keys.PREFERRED_BLOCK_SIZE,32); + * #config.setInt(NetworkConfig.Keys.MAX_RESOURCE_BODY_SIZE,2048); + * #config.setInt(NetworkConfig.Keys.MAX_RETRANSMIT,3); + * #config.setInt(NetworkConfig.Keys.MAX_TRANSMIT_WAIT,120000); + */ + + /** + * Property to indicate if the response should always include the Block2 option \ + * when client request early blockwise negociation but the response can be sent on one packet. + * - value of false indicate that the server will respond without block2 option if no further blocks are required. + * - value of true indicate that the server will response with block2 option event if no further blocks are required. + * CoAP client will try to use block mode + * or adapt the block size when receiving a 4.13 Entity too large response code + */ + coapConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_STRICT_BLOCK2_OPTION, true); + /*** + * Property to indicate if the response should always include the Block2 option \ + * when client request early blockwise negociation but the response can be sent on one packet. + * - value of false indicate that the server will respond without block2 option if no further blocks are required. + * - value of true indicate that the server will response with block2 option event if no further blocks are required. + */ + coapConfig.setBoolean(NetworkConfig.Keys.BLOCKWISE_ENTITY_TOO_LARGE_AUTO_FAILOVER, true); + + coapConfig.setInt(NetworkConfig.Keys.BLOCKWISE_STATUS_LIFETIME, 300000); + /** + * !!! REQUEST_ENTITY_TOO_LARGE CODE=4.13 + * The maximum size of a resource body (in bytes) that will be accepted + * as the payload of a POST/PUT or the response to a GET request in a + * transparent> blockwise transfer. + * This option serves as a safeguard against excessive memory + * consumption when many resources contain large bodies that cannot be + * transferred in a single CoAP message. This option has no impact on + * *manually* managed blockwise transfers in which the blocks are handled individually. + * Note that this option does not prevent local clients or resource + * implementations from sending large bodies as part of a request or response to a peer. + * The default value of this property is DEFAULT_MAX_RESOURCE_BODY_SIZE = 8192 + * A value of {@code 0} turns off transparent handling of blockwise transfers altogether. + */ +// coapConfig.setInt(NetworkConfig.Keys.MAX_RESOURCE_BODY_SIZE, 8192); + coapConfig.setInt(NetworkConfig.Keys.MAX_RESOURCE_BODY_SIZE, 16384); + /** + * The default DTLS response matcher. + * Supported values are STRICT, RELAXED, or PRINCIPAL. + * The default value is STRICT. + * Create new instance of udp endpoint context matcher. + * Params: + * checkAddress + * – true with address check, (STRICT, UDP) + * - false, without + */ + coapConfig.setString(NetworkConfig.Keys.RESPONSE_MATCHING, "STRICT"); + /** + * https://tools.ietf.org/html/rfc7959#section-2.9.3 + * The block size (number of bytes) to use when doing a blockwise transfer. \ + * This value serves as the upper limit for block size in blockwise transfers + */ + coapConfig.setInt(NetworkConfig.Keys.PREFERRED_BLOCK_SIZE, 512); + /** + * The maximum payload size (in bytes) that can be transferred in a + * single message, i.e. without requiring a blockwise transfer. + * NB: this value MUST be adapted to the maximum message size supported by the transport layer. + * In particular, this value cannot exceed the network's MTU if UDP is used as the transport protocol + * DEFAULT_VALUE = 1024 + */ + coapConfig.setInt(NetworkConfig.Keys.MAX_MESSAGE_SIZE, 512); + + coapConfig.setInt(NetworkConfig.Keys.MAX_RETRANSMIT, 4); + + return coapConfig; + } +} diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java index c17e37fd4a..72626f1103 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mServerListener.java @@ -26,6 +26,7 @@ import org.eclipse.leshan.server.registration.RegistrationUpdate; import java.util.Collection; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.LOG_LW2M_INFO; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.convertPathFromObjectIdToIdVer; @Slf4j @@ -85,17 +86,19 @@ public class LwM2mServerListener { @Override public void cancelled(Observation observation) { - log.info("Received notification cancelled from [{}] ", observation.getPath()); + String msg = String.format("%s: Cancel Observation %s.", LOG_LW2M_INFO, observation.getPath()); + service.sendLogsToThingsboard(msg, observation.getRegistrationId()); + log.trace(msg); } @Override public void onResponse(Observation observation, Registration registration, ObserveResponse response) { if (registration != null) { try { - service.onObservationResponse(registration, convertPathFromObjectIdToIdVer(observation.getPath().toString(), + service.onUpdateValueAfterReadResponse(registration, convertPathFromObjectIdToIdVer(observation.getPath().toString(), registration), response, null); } catch (Exception e) { - log.error("[{}] onResponse", e.toString()); + log.error("Observation/Read onResponse", e); } } @@ -108,7 +111,10 @@ public class LwM2mServerListener { @Override public void newObservation(Observation observation, Registration registration) { - log.info("Received newObservation from [{}] endpoint [{}] ", observation.getPath(), registration.getEndpoint()); + String msg = String.format("%s: Successful start newObservation %s.", LOG_LW2M_INFO, + observation.getPath()); + service.sendLogsToThingsboard(msg, registration.getId()); + log.trace(msg); } }; diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportHandler.java index ac5adcb7a3..08e46eb288 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportHandler.java @@ -25,7 +25,6 @@ import com.google.gson.JsonSyntaxException; import com.google.gson.reflect.TypeToken; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.eclipse.californium.core.network.config.NetworkConfig; import org.eclipse.leshan.core.attributes.Attribute; import org.eclipse.leshan.core.attributes.AttributeSet; import org.eclipse.leshan.core.model.ObjectModel; @@ -40,7 +39,6 @@ import org.eclipse.leshan.core.node.codec.CodecException; import org.eclipse.leshan.core.request.DownlinkRequest; import org.eclipse.leshan.core.request.WriteAttributesRequest; import org.eclipse.leshan.core.util.Hex; -import org.eclipse.leshan.server.californium.LeshanServerBuilder; import org.eclipse.leshan.server.registration.Registration; import org.nustaq.serialization.FSTConfiguration; import org.thingsboard.server.common.data.DeviceProfile; @@ -50,7 +48,6 @@ import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClientProfile; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -72,7 +69,6 @@ import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPA @Slf4j public class LwM2mTransportHandler { - // public static final String BASE_DEVICE_API_TOPIC = "v1/devices/me"; public static final String TRANSPORT_DEFAULT_LWM2M_VERSION = "1.0"; public static final String CLIENT_LWM2M_SETTINGS = "clientLwM2mSettings"; public static final String BOOTSTRAP = "bootstrap"; @@ -85,19 +81,12 @@ public class LwM2mTransportHandler { public static final String KEY_NAME = "keyName"; public static final String OBSERVE_LWM2M = "observe"; public static final String ATTRIBUTE_LWM2M = "attributeLwm2m"; -// public static final String RESOURCE_VALUE = "resValue"; -// public static final String RESOURCE_TYPE = "resType"; private static final String REQUEST = "/request"; - // private static final String RESPONSE = "/response"; private static final String ATTRIBUTES = "/" + ATTRIBUTE; public static final String TELEMETRIES = "/" + TELEMETRY; - // public static final String ATTRIBUTES_RESPONSE = ATTRIBUTES + RESPONSE; public static final String ATTRIBUTES_REQUEST = ATTRIBUTES + REQUEST; - // public static final String DEVICE_ATTRIBUTES_RESPONSE = ATTRIBUTES_RESPONSE + "/"; public static final String DEVICE_ATTRIBUTES_REQUEST = ATTRIBUTES_REQUEST + "/"; -// public static final String DEVICE_ATTRIBUTES_TOPIC = BASE_DEVICE_API_TOPIC + ATTRIBUTES; -// public static final String DEVICE_TELEMETRY_TOPIC = BASE_DEVICE_API_TOPIC + TELEMETRIES; public static final long DEFAULT_TIMEOUT = 2 * 60 * 1000L; // 2min in ms @@ -112,6 +101,11 @@ public class LwM2mTransportHandler { public static final String CLIENT_NOT_AUTHORIZED = "Client not authorized"; + public static final Integer FR_OBJECT_ID = 5; + public static final Integer FR_RESOURCE_VER_ID = 7; + public static final String FR_PATH_RESOURCE_VER_ID = LWM2M_SEPARATOR_PATH + FR_OBJECT_ID + LWM2M_SEPARATOR_PATH + + "0" + LWM2M_SEPARATOR_PATH + FR_RESOURCE_VER_ID; + public enum LwM2mTypeServer { BOOTSTRAP(0, "bootstrap"), CLIENT(1, "client"); @@ -168,6 +162,8 @@ public class LwM2mTransportHandler { WRITE_ATTRIBUTES(8, "WriteAttributes"), DELETE(9, "Delete"); +// READ_INFO_FW(10, "ReadInfoFirmware"); + public int code; public String type; @@ -190,21 +186,6 @@ public class LwM2mTransportHandler { public static final String SERVICE_CHANNEL = "SERVICE"; public static final String RESPONSE_CHANNEL = "RESP"; - public static NetworkConfig getCoapConfig(Integer serverPortNoSec, Integer serverSecurePort) { - NetworkConfig coapConfig; - File configFile = new File(NetworkConfig.DEFAULT_FILE_NAME); - if (configFile.isFile()) { - coapConfig = new NetworkConfig(); - coapConfig.load(configFile); - } else { - coapConfig = LeshanServerBuilder.createDefaultNetworkConfig(); - coapConfig.store(configFile); - } - coapConfig.setString("COAP_PORT", Integer.toString(serverPortNoSec)); - coapConfig.setString("COAP_SECURE_PORT", Integer.toString(serverSecurePort)); - return coapConfig; - } - public static boolean equalsResourceValue(Object valueOld, Object valueNew, ResourceModel.Type type, LwM2mPath resourcePath) throws CodecException { switch (type) { case BOOLEAN: diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportRequest.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportRequest.java index 9d7c22a112..370abaa9cd 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportRequest.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportRequest.java @@ -67,6 +67,7 @@ import static org.eclipse.californium.core.coap.CoAP.ResponseCode.CONTENT; import static org.eclipse.leshan.core.ResponseCode.BAD_REQUEST; import static org.eclipse.leshan.core.ResponseCode.NOT_FOUND; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.DEFAULT_TIMEOUT; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.FR_PATH_RESOURCE_VER_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.LOG_LW2M_ERROR; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.LOG_LW2M_INFO; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.LOG_LW2M_VALUE; @@ -125,7 +126,6 @@ public class LwM2mTransportRequest { public void sendAllRequest(Registration registration, String targetIdVer, LwM2mTypeOper typeOper, String contentFormatName, Object params, long timeoutInMs, Lwm2mClientRpcRequest rpcRequest) { try { - String target = convertPathFromIdVerToObjectId(targetIdVer); DownlinkRequest request = null; ContentFormat contentFormat = contentFormatName != null ? ContentFormat.fromName(contentFormatName.toUpperCase()) : ContentFormat.DEFAULT; @@ -145,11 +145,11 @@ public class LwM2mTransportRequest { break; case OBSERVE: if (resultIds.isResource()) { - request = new ObserveRequest(resultIds.getObjectId(), resultIds.getObjectInstanceId(), resultIds.getResourceId()); + request = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId(), resultIds.getResourceId()); } else if (resultIds.isObjectInstance()) { - request = new ObserveRequest(resultIds.getObjectId(), resultIds.getObjectInstanceId()); + request = new ObserveRequest(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId()); } else if (resultIds.getObjectId() >= 0) { - request = new ObserveRequest(resultIds.getObjectId()); + request = new ObserveRequest(contentFormat, resultIds.getObjectId()); } break; case OBSERVE_CANCEL: @@ -171,8 +171,6 @@ public class LwM2mTransportRequest { break; case WRITE_REPLACE: // Request to write a String Single-Instance Resource using the TLV content format. -// resource = lwM2MClient.getResourceModel(targetIdVer); -// if (contentFormat.equals(ContentFormat.TLV) && !resource.multiple) { resourceModel = lwM2MClient.getResourceModel(targetIdVer, this.lwM2mTransportContextServer.getLwM2MTransportConfigServer() .getModelProvider()); if (contentFormat.equals(ContentFormat.TLV)) { @@ -181,7 +179,6 @@ public class LwM2mTransportRequest { registration, rpcRequest); } // Mode.REPLACE && Request to write a String Single-Instance Resource using the given content format (TEXT, TLV, JSON) -// else if (!contentFormat.equals(ContentFormat.TLV) && !resource.multiple) { else if (!contentFormat.equals(ContentFormat.TLV)) { request = this.getWriteRequestSingleResource(contentFormat, resultIds.getObjectId(), resultIds.getObjectInstanceId(), resultIds.getResourceId(), params, resourceModel.type, @@ -215,13 +212,16 @@ public class LwM2mTransportRequest { long finalTimeoutInMs = timeoutInMs; lwM2MClient.getQueuedRequests().add(() -> sendRequest(registration, lwM2MClient, finalRequest, finalTimeoutInMs, rpcRequest)); } catch (Exception e) { - log.error("[{}] [{}] [{}] Failed to send downlink.", registration.getEndpoint(), targetIdVer, typeOper, e); + log.error("[{}] [{}] [{}] Failed to send downlink.", registration.getEndpoint(), targetIdVer, typeOper.name(), e); + } + } else if (OBSERVE_CANCEL == typeOper) { + log.trace("[{}], [{}] - [{}] SendRequest", registration.getEndpoint(), typeOper.name(), targetIdVer); + if (rpcRequest != null) { + rpcRequest.setInfoMsg(null); + serviceImpl.sentRpcRequest(rpcRequest, CONTENT.name(), null, null); } - } else if (OBSERVE_CANCEL == typeOper && rpcRequest != null) { - rpcRequest.setInfoMsg(null); - serviceImpl.sentRpcRequest(rpcRequest, CONTENT.name(), null, null); } else { - log.error("[{}], [{}] - [{}] error SendRequest", registration.getEndpoint(), typeOper, targetIdVer); + log.error("[{}], [{}] - [{}] error SendRequest", registration.getEndpoint(), typeOper.name(), targetIdVer); if (rpcRequest != null) { String errorMsg = resourceModel == null ? String.format("Path %s not found in object version", targetIdVer) : "SendRequest - null"; serviceImpl.sentRpcRequest(rpcRequest, NOT_FOUND.getName(), errorMsg, LOG_LW2M_ERROR); @@ -236,8 +236,8 @@ public class LwM2mTransportRequest { Set observationPaths = observations.stream().map(observation -> observation.getPath().toString()).collect(Collectors.toUnmodifiableSet()); String msg = String.format("%s: type operation %s observation paths - %s", LOG_LW2M_INFO, OBSERVE_READ_ALL.type, observationPaths); - serviceImpl.sendLogsToThingsboard(msg, registration); - log.info("[{}], [{}]", registration.getEndpoint(), msg); + serviceImpl.sendLogsToThingsboard(msg, registration.getId()); + log.trace("[{}] [{}], [{}]", typeOper.name(), registration.getEndpoint(), msg); if (rpcRequest != null) { String valueMsg = String.format("Observation paths - %s", observationPaths); serviceImpl.sentRpcRequest(rpcRequest, CONTENT.name(), valueMsg, LOG_LW2M_VALUE); @@ -246,7 +246,7 @@ public class LwM2mTransportRequest { } catch (Exception e) { String msg = String.format("%s: type operation %s %s", LOG_LW2M_ERROR, typeOper.name(), e.getMessage()); - serviceImpl.sendLogsToThingsboard(msg, registration); + serviceImpl.sendLogsToThingsboard(msg, registration.getId()); throw new Exception(e); } } @@ -261,45 +261,53 @@ public class LwM2mTransportRequest { private void sendRequest(Registration registration, LwM2mClient lwM2MClient, DownlinkRequest request, long timeoutInMs, Lwm2mClientRpcRequest rpcRequest) { leshanServer.send(registration, request, timeoutInMs, (ResponseCallback) response -> { if (!lwM2MClient.isInit()) { - lwM2MClient.initValue(this.serviceImpl, convertPathFromObjectIdToIdVer(request.getPath().toString(), registration)); + lwM2MClient.initReadValue(this.serviceImpl, convertPathFromObjectIdToIdVer(request.getPath().toString(), registration)); } if (CoAP.ResponseCode.isSuccess(((Response) response.getCoapResponse()).getCode())) { this.handleResponse(registration, request.getPath().toString(), response, request, rpcRequest); - if (request instanceof WriteRequest && ((WriteRequest) request).isReplaceRequest()) { - LwM2mNode node = ((WriteRequest) request).getNode(); - Object value = this.converter.convertValue(((LwM2mSingleResource) node).getValue(), - ((LwM2mSingleResource) node).getType(), ResourceModel.Type.STRING, request.getPath()); - String msg = String.format("%s: sendRequest Replace: CoapCde - %s Lwm2m code - %d name - %s Resource path - %s value - %s SendRequest to Client", - LOG_LW2M_INFO, ((Response) response.getCoapResponse()).getCode(), response.getCode().getCode(), - response.getCode().getName(), request.getPath().toString(), value); - serviceImpl.sendLogsToThingsboard(msg, registration); - log.info("[{}] [{}] - [{}] [{}] Update SendRequest[{}]", registration.getEndpoint(), - ((Response) response.getCoapResponse()).getCode(), response.getCode(), - request.getPath().toString(), value); - serviceImpl.sentRpcRequest(rpcRequest, response.getCode().getName(), null, LOG_LW2M_INFO); - } } else { - String msg = String.format("%s: sendRequest: CoapCode - %s Lwm2m code - %d name - %s Resource path - %s SendRequest to Client", LOG_LW2M_ERROR, + String msg = String.format("%s: SendRequest %s: CoapCode - %s Lwm2m code - %d name - %s Resource path - %s", LOG_LW2M_ERROR, request.getClass().getName().toString(), ((Response) response.getCoapResponse()).getCode(), response.getCode().getCode(), response.getCode().getName(), request.getPath().toString()); - serviceImpl.sendLogsToThingsboard(msg, registration); - log.error("[{}], [{}] - [{}] [{}] error SendRequest", registration.getEndpoint(), ((Response) response.getCoapResponse()).getCode(), response.getCode(), request.getPath().toString()); + serviceImpl.sendLogsToThingsboard(msg, registration.getId()); + log.error("[{}] [{}], [{}] - [{}] [{}] error SendRequest", request.getClass().getName().toString(), registration.getEndpoint(), + ((Response) response.getCoapResponse()).getCode(), response.getCode(), request.getPath().toString()); + if (!lwM2MClient.isInit()) { + lwM2MClient.initReadValue(this.serviceImpl, convertPathFromObjectIdToIdVer(request.getPath().toString(), registration)); + } if (rpcRequest != null) { serviceImpl.sentRpcRequest(rpcRequest, response.getCode().getName(), response.getErrorMessage(), LOG_LW2M_ERROR); } + /** Not Found + * set setClient_fw_version = empty + **/ + if (FR_PATH_RESOURCE_VER_ID.equals(request.getPath().toString()) && lwM2MClient.isUpdateFw()) { + lwM2MClient.setUpdateFw(false); + lwM2MClient.getFrUpdate().setClientFwVersion(""); + log.warn("updateFirmwareClient1"); + serviceImpl.updateFirmwareClient(lwM2MClient); + } } }, e -> { + /** version == null + * set setClient_fw_version = empty + **/ + if (FR_PATH_RESOURCE_VER_ID.equals(request.getPath().toString()) && lwM2MClient.isUpdateFw()) { + lwM2MClient.setUpdateFw(false); + lwM2MClient.getFrUpdate().setClientFwVersion(""); + log.warn("updateFirmwareClient2"); + serviceImpl.updateFirmwareClient(lwM2MClient); + } if (!lwM2MClient.isInit()) { - lwM2MClient.initValue(this.serviceImpl, convertPathFromObjectIdToIdVer(request.getPath().toString(), registration)); + lwM2MClient.initReadValue(this.serviceImpl, convertPathFromObjectIdToIdVer(request.getPath().toString(), registration)); } - String msg = String.format("%s: sendRequest: Resource path - %s msg error - %s SendRequest to Client", - LOG_LW2M_ERROR, request.getPath().toString(), e.getMessage()); - serviceImpl.sendLogsToThingsboard(msg, registration); - log.error("[{}] - [{}] error SendRequest", request.getPath().toString(), e.toString()); + String msg = String.format("%s: SendRequest %s: Resource path - %s msg error - %s", + LOG_LW2M_ERROR, request.getClass().getName().toString(), request.getPath().toString(), e.getMessage()); + serviceImpl.sendLogsToThingsboard(msg, registration.getId()); + log.error("[{}] [{}] - [{}] error SendRequest", request.getClass().getName().toString(), request.getPath().toString(), e.toString()); if (rpcRequest != null) { serviceImpl.sentRpcRequest(rpcRequest, CoAP.CodeClass.ERROR_RESPONSE.name(), e.getMessage(), LOG_LW2M_ERROR); } }); - } private WriteRequest getWriteRequestSingleResource(ContentFormat contentFormat, Integer objectId, Integer instanceId, @@ -323,7 +331,9 @@ public class LwM2mTransportRequest { Date date = new Date(Long.decode(value.toString())); return (contentFormat == null) ? new WriteRequest(objectId, instanceId, resourceId, date) : new WriteRequest(contentFormat, objectId, instanceId, resourceId, date); case OPAQUE: // byte[] value, base64 - return (contentFormat == null) ? new WriteRequest(objectId, instanceId, resourceId, Hex.decodeHex(value.toString().toCharArray())) : new WriteRequest(contentFormat, objectId, instanceId, resourceId, Hex.decodeHex(value.toString().toCharArray())); + byte[] valueRequest = value instanceof byte[] ? (byte[]) value : Hex.decodeHex(value.toString().toCharArray()); + return (contentFormat == null) ? new WriteRequest(objectId, instanceId, resourceId, valueRequest) : + new WriteRequest(contentFormat, objectId, instanceId, resourceId, valueRequest); default: } } @@ -337,7 +347,7 @@ public class LwM2mTransportRequest { String patn = "/" + objectId + "/" + instanceId + "/" + resourceId; String msg = String.format(LOG_LW2M_ERROR + ": NumberFormatException: Resource path - %s type - %s value - %s msg error - %s SendRequest to Client", patn, type, value, e.toString()); - serviceImpl.sendLogsToThingsboard(msg, registration); + serviceImpl.sendLogsToThingsboard(msg, registration.getId()); log.error("Path: [{}] type: [{}] value: [{}] errorMsg: [{}]]", patn, type, value, e.toString()); if (rpcRequest != null) { String errorMsg = String.format("NumberFormatException: Resource path - %s type - %s value - %s", patn, type, value); @@ -369,7 +379,7 @@ public class LwM2mTransportRequest { DownlinkRequest request, Lwm2mClientRpcRequest rpcRequest) { String pathIdVer = convertPathFromObjectIdToIdVer(path, registration); if (response instanceof ReadResponse) { - serviceImpl.onObservationResponse(registration, pathIdVer, (ReadResponse) response, rpcRequest); + serviceImpl.onUpdateValueAfterReadResponse(registration, pathIdVer, (ReadResponse) response, rpcRequest); } else if (response instanceof CancelObservationResponse) { log.info("[{}] Path [{}] CancelObservationResponse 3_Send", pathIdVer, response); @@ -389,14 +399,33 @@ public class LwM2mTransportRequest { } else if (response instanceof WriteAttributesResponse) { log.info("[{}] Path [{}] WriteAttributesResponse 8_Send", pathIdVer, response); } else if (response instanceof WriteResponse) { - log.info("[{}] Path [{}] WriteAttributesResponse 9_Send", pathIdVer, response); + log.info("[{}] Path [{}] WriteResponse 9_Send", pathIdVer, response); + this.infoWriteResponse(registration, response, request); serviceImpl.onWriteResponseOk(registration, pathIdVer, (WriteRequest) request); } - if (rpcRequest != null && (response instanceof ExecuteResponse - || response instanceof WriteAttributesResponse - || response instanceof DeleteResponse)) { - rpcRequest.setInfoMsg(null); - serviceImpl.sentRpcRequest(rpcRequest, response.getCode().getName(), null, null); + if (rpcRequest != null) { + if (response instanceof ExecuteResponse + || response instanceof WriteAttributesResponse + || response instanceof DeleteResponse) { + rpcRequest.setInfoMsg(null); + serviceImpl.sentRpcRequest(rpcRequest, response.getCode().getName(), null, null); + } else if (response instanceof WriteResponse) { + serviceImpl.sentRpcRequest(rpcRequest, response.getCode().getName(), null, LOG_LW2M_INFO); + } } } + + private void infoWriteResponse(Registration registration, LwM2mResponse response, + DownlinkRequest request) { + LwM2mNode node = ((WriteRequest) request).getNode(); + Object value = this.converter.convertValue(((LwM2mSingleResource) node).getValue(), + ((LwM2mSingleResource) node).getType(), ResourceModel.Type.STRING, request.getPath()); + String msg = String.format("%s: Update finished successfully: CoapCde - %s Lwm2m code - %d name - %s Resource path - %s value - %s", + LOG_LW2M_INFO, ((Response) response.getCoapResponse()).getCode(), response.getCode().getCode(), + response.getCode().getName(), request.getPath().toString(), value); + serviceImpl.sendLogsToThingsboard(msg, registration.getId()); + log.warn("[{}] [{}] [{}] - [{}] [{}] Update finished successfully: [{}]", request.getClass().getName().toString(), registration.getEndpoint(), + ((Response) response.getCoapResponse()).getCode(), response.getCode(), + request.getPath().toString(), value); + } } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerConfiguration.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerConfiguration.java index 0af20b3e51..86bec24984 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerConfiguration.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServerConfiguration.java @@ -16,6 +16,8 @@ package org.thingsboard.server.transport.lwm2m.server; import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.network.config.NetworkConfig; +import org.eclipse.californium.core.network.stack.BlockwiseLayer; import org.eclipse.californium.scandium.config.DtlsConnectorConfig; import org.eclipse.leshan.core.node.codec.DefaultLwM2mNodeDecoder; import org.eclipse.leshan.core.node.codec.DefaultLwM2mNodeEncoder; @@ -57,7 +59,7 @@ import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_ECDHE import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8; import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_PSK_WITH_AES_128_CBC_SHA256; import static org.eclipse.californium.scandium.dtls.cipher.CipherSuite.TLS_PSK_WITH_AES_128_CCM_8; -import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.getCoapConfig; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mNetworkConfig.getCoapConfig; @Slf4j @Component @@ -92,7 +94,10 @@ public class LwM2mTransportServerConfiguration { /** Use a magic converter to support bad type send by the UI. */ builder.setEncoder(new DefaultLwM2mNodeEncoder(LwM2mValueConverterImpl.getInstance())); + /** Create CoAP Config */ + NetworkConfig networkConfig = getCoapConfig(serverPortNoSec, serverSecurePort); + BlockwiseLayer blockwiseLayer = new BlockwiseLayer(networkConfig); builder.setCoapConfig(getCoapConfig(serverPortNoSec, serverSecurePort)); /** Define model provider (Create Models )*/ @@ -110,6 +115,7 @@ public class LwM2mTransportServerConfiguration { /** Create DTLS Config */ DtlsConnectorConfig.Builder dtlsConfig = new DtlsConnectorConfig.Builder(); + dtlsConfig.setServerOnly(true); dtlsConfig.setRecommendedSupportedGroupsOnly(this.context.getLwM2MTransportConfigServer().isRecommendedSupportedGroups()); dtlsConfig.setRecommendedCipherSuitesOnly(this.context.getLwM2MTransportConfigServer().isRecommendedCiphers()); if (this.pskMode) { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java index 91588c0490..dec1e7889d 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportService.java @@ -39,7 +39,7 @@ public interface LwM2mTransportService extends TbTransportService { void setCancelObservations(Registration registration); - void onObservationResponse(Registration registration, String path, ReadResponse response, Lwm2mClientRpcRequest rpcRequest); + void onUpdateValueAfterReadResponse(Registration registration, String path, ReadResponse response, Lwm2mClientRpcRequest rpcRequest); void onAttributeUpdate(TransportProtos.AttributeUpdateNotificationMsg msg, TransportProtos.SessionInfoProto sessionInfo); @@ -60,6 +60,4 @@ public interface LwM2mTransportService extends TbTransportService { void doTrigger(Registration registration, String path); void doDisconnect(TransportProtos.SessionInfoProto sessionInfo); - - } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java index 31ae4bdf83..f52919ea3c 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/LwM2mTransportServiceImpl.java @@ -38,10 +38,12 @@ import org.eclipse.leshan.server.registration.Registration; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.firmware.FirmwareDataCache; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.transport.TransportService; +import org.thingsboard.server.common.transport.TransportServiceCallback; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; @@ -77,9 +79,12 @@ import java.util.stream.Collectors; import static org.eclipse.californium.core.coap.CoAP.ResponseCode.BAD_REQUEST; import static org.eclipse.leshan.core.attributes.Attribute.OBJECT_VERSION; +import static org.thingsboard.server.common.data.DataConstants.FIRMWARE_VERSION; +import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_KEY; import static org.thingsboard.server.common.data.lwm2m.LwM2mConstants.LWM2M_SEPARATOR_PATH; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.CLIENT_NOT_AUTHORIZED; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.DEVICE_ATTRIBUTES_REQUEST; +import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.FR_PATH_RESOURCE_VER_ID; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.LOG_LW2M_ERROR; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.LOG_LW2M_INFO; import static org.thingsboard.server.transport.lwm2m.server.LwM2mTransportHandler.LOG_LW2M_VALUE; @@ -110,6 +115,8 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { private ExecutorService executorUpdateRegistered; private ExecutorService executorUnRegistered; private LwM2mValueConverterImpl converter; + private FirmwareDataCache firmwareDataCache; + private final TransportService transportService; @@ -121,12 +128,15 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { private final LwM2mTransportRequest lwM2mTransportRequest; - public LwM2mTransportServiceImpl(TransportService transportService, LwM2mTransportContextServer lwM2mTransportContextServer, LwM2mClientContext lwM2mClientContext, LeshanServer leshanServer, @Lazy LwM2mTransportRequest lwM2mTransportRequest) { + public LwM2mTransportServiceImpl(TransportService transportService, LwM2mTransportContextServer lwM2mTransportContextServer, + LwM2mClientContext lwM2mClientContext, LeshanServer leshanServer, + @Lazy LwM2mTransportRequest lwM2mTransportRequest, FirmwareDataCache firmwareDataCache) { this.transportService = transportService; this.lwM2mTransportContextServer = lwM2mTransportContextServer; this.lwM2mClientContext = lwM2mClientContext; this.leshanServer = leshanServer; this.lwM2mTransportRequest = lwM2mTransportRequest; + this.firmwareDataCache = firmwareDataCache; } @PostConstruct @@ -168,8 +178,9 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { transportService.process(sessionInfo, DefaultTransportService.getSessionEventMsg(SessionEvent.OPEN), null); transportService.process(sessionInfo, TransportProtos.SubscribeToAttributeUpdatesMsg.newBuilder().build(), null); transportService.process(sessionInfo, TransportProtos.SubscribeToRPCMsg.newBuilder().build(), null); + this.getInfoFirmwareUpdate(lwM2MClient); this.initLwM2mFromClientValue(registration, lwM2MClient); - this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client create after Registration", registration); + this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client create after Registration", registration.getId()); } else { log.error("Client: [{}] onRegistered [{}] name [{}] sessionInfo ", registration.getId(), registration.getEndpoint(), null); } @@ -224,7 +235,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { executorUnRegistered.submit(() -> { try { this.setCancelObservations(registration); - this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client unRegistration", registration); + this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client unRegistration", registration.getId()); this.closeClientSession(registration); } catch (Throwable t) { log.error("[{}] endpoint [{}] error Unable un registration.", registration.getEndpoint(), t); @@ -257,7 +268,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { @Override public void onSleepingDev(Registration registration) { log.info("[{}] [{}] Received endpoint Sleeping version event", registration.getId(), registration.getEndpoint()); - this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client is sleeping!", registration); + this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client is sleeping!", registration.getId()); //TODO: associate endpointId with device information. } @@ -280,7 +291,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { * @param response - observe */ @Override - public void onObservationResponse(Registration registration, String path, ReadResponse response, Lwm2mClientRpcRequest rpcRequest) { + public void onUpdateValueAfterReadResponse(Registration registration, String path, ReadResponse response, Lwm2mClientRpcRequest rpcRequest) { if (response.getContent() != null) { if (response.getContent() instanceof LwM2mObject) { LwM2mObject lwM2mObject = (LwM2mObject) response.getContent(); @@ -304,20 +315,26 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { /** * Update - send request in change value resources in Client - * Path to resources from profile equal keyName or from ModelObject equal name - * Only for resources: isWritable && isPresent as attribute in profile -> LwM2MClientProfile (format: CamelCase) - * Delete - nothing * + * 1. FirmwareUpdate: + * - If msg.getSharedUpdatedList().forEach(tsKvProto -> {tsKvProto.getKv().getKey().indexOf(FIRMWARE_UPDATE_PREFIX, 0) == 0 + * 2. Shared Other AttributeUpdate + * -- Path to resources from profile equal keyName or from ModelObject equal name + * -- Only for resources: isWritable && isPresent as attribute in profile -> LwM2MClientProfile (format: CamelCase) + * 3. Delete - nothing * * @param msg - */ @Override public void onAttributeUpdate(AttributeUpdateNotificationMsg msg, TransportProtos.SessionInfoProto sessionInfo) { + LwM2mClient lwM2MClient = lwM2mClientContext.getLwM2mClient(new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB())); if (msg.getSharedUpdatedCount() > 0) { msg.getSharedUpdatedList().forEach(tsKvProto -> { String pathName = tsKvProto.getKv().getKey(); String pathIdVer = this.getPresentPathIntoProfile(sessionInfo, pathName); Object valueNew = this.lwM2mTransportContextServer.getValueFromKvProto(tsKvProto.getKv()); - LwM2mClient lwM2MClient = lwM2mClientContext.getLwM2mClient(new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB())); + if (FIRMWARE_VERSION.equals(pathName) && !valueNew.equals(lwM2MClient.getFrUpdate().getCurrentFwVersion())) { + this.getInfoFirmwareUpdate(lwM2MClient); + } if (pathIdVer != null) { ResourceModel resourceModel = lwM2MClient.getResourceModel(pathIdVer, this.lwM2mTransportContextServer.getLwM2MTransportConfigServer() .getModelProvider()); @@ -327,19 +344,26 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { log.error("Resource path - [{}] value - [{}] is not Writable and cannot be updated", pathIdVer, valueNew); String logMsg = String.format("%s: attributeUpdate: Resource path - %s value - %s is not Writable and cannot be updated", LOG_LW2M_ERROR, pathIdVer, valueNew); - this.sendLogsToThingsboard(logMsg, lwM2MClient.getRegistration()); + this.sendLogsToThingsboard(logMsg, lwM2MClient.getRegistration().getId()); } } else { log.error("Resource name name - [{}] value - [{}] is not present as attribute/telemetry in profile and cannot be updated", pathName, valueNew); String logMsg = String.format("%s: attributeUpdate: attribute name - %s value - %s is not present as attribute in profile and cannot be updated", LOG_LW2M_ERROR, pathName, valueNew); - this.sendLogsToThingsboard(logMsg, lwM2MClient.getRegistration()); + this.sendLogsToThingsboard(logMsg, lwM2MClient.getRegistration().getId()); } + }); } else if (msg.getSharedDeletedCount() > 0) { + msg.getSharedUpdatedList().forEach(tsKvProto -> { + String pathName = tsKvProto.getKv().getKey(); + Object valueNew = this.lwM2mTransportContextServer.getValueFromKvProto(tsKvProto.getKv()); + if (FIRMWARE_VERSION.equals(pathName) && !valueNew.equals(lwM2MClient.getFrUpdate().getCurrentFwVersion())) { + lwM2MClient.getFrUpdate().setCurrentFwVersion((String) valueNew); + } + }); log.info("[{}] delete [{}] onAttributeUpdate", msg.getSharedDeletedList(), sessionInfo); } - } /** @@ -455,23 +479,21 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { } if (rpcRequest.has(lwm2mClientRpcRequest.paramsKey) && rpcRequest.get(lwm2mClientRpcRequest.paramsKey).isJsonObject()) { lwm2mClientRpcRequest.setParams(new Gson().fromJson(rpcRequest.get(lwm2mClientRpcRequest.paramsKey) - .getAsJsonObject().toString(), new TypeToken>() { - }.getType())); + .getAsJsonObject().toString(), new TypeToken>() { + }.getType())); } lwm2mClientRpcRequest.setSessionInfo(sessionInfo); if (OBSERVE_READ_ALL != lwm2mClientRpcRequest.getTypeOper() && lwm2mClientRpcRequest.getTargetIdVer() == null) { lwm2mClientRpcRequest.setErrorMsg(lwm2mClientRpcRequest.targetIdVerKey + " and " + lwm2mClientRpcRequest.keyNameKey + " is null or bad format"); - } - else if ((EXECUTE == lwm2mClientRpcRequest.getTypeOper() + } else if ((EXECUTE == lwm2mClientRpcRequest.getTypeOper() || WRITE_REPLACE == lwm2mClientRpcRequest.getTypeOper()) - && lwm2mClientRpcRequest.getTargetIdVer() !=null + && lwm2mClientRpcRequest.getTargetIdVer() != null && !(new LwM2mPath(convertPathFromIdVerToObjectId(lwm2mClientRpcRequest.getTargetIdVer())).isResource() || new LwM2mPath(convertPathFromIdVerToObjectId(lwm2mClientRpcRequest.getTargetIdVer())).isResourceInstance())) { - lwm2mClientRpcRequest.setErrorMsg("Invalid parameter " + lwm2mClientRpcRequest.targetIdVerKey + lwm2mClientRpcRequest.setErrorMsg("Invalid parameter " + lwm2mClientRpcRequest.targetIdVerKey + ". Only Resource or ResourceInstance can be this operation"); - } - else if (WRITE_UPDATE == lwm2mClientRpcRequest.getTypeOper()){ + } else if (WRITE_UPDATE == lwm2mClientRpcRequest.getTypeOper()) { lwm2mClientRpcRequest.setErrorMsg("Procedures In Development..."); } } else { @@ -483,23 +505,23 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { return lwm2mClientRpcRequest; } - public void sentRpcRequest (Lwm2mClientRpcRequest rpcRequest, String requestCode, String msg, String typeMsg) { + public void sentRpcRequest(Lwm2mClientRpcRequest rpcRequest, String requestCode, String msg, String typeMsg) { rpcRequest.setResponseCode(requestCode); - if (LOG_LW2M_ERROR.equals(typeMsg)) { - rpcRequest.setInfoMsg(null); - rpcRequest.setValueMsg(null); - if (rpcRequest.getErrorMsg() == null) { - msg = msg.isEmpty() ? null : msg; - rpcRequest.setErrorMsg(msg); - } - } else if (LOG_LW2M_INFO.equals(typeMsg)) { - if (rpcRequest.getInfoMsg() == null) { - rpcRequest.setInfoMsg(msg); - } - } else if (LOG_LW2M_VALUE.equals(typeMsg)) { - if (rpcRequest.getValueMsg() == null) { - rpcRequest.setValueMsg(msg); - } + if (LOG_LW2M_ERROR.equals(typeMsg)) { + rpcRequest.setInfoMsg(null); + rpcRequest.setValueMsg(null); + if (rpcRequest.getErrorMsg() == null) { + msg = msg.isEmpty() ? null : msg; + rpcRequest.setErrorMsg(msg); + } + } else if (LOG_LW2M_INFO.equals(typeMsg)) { + if (rpcRequest.getInfoMsg() == null) { + rpcRequest.setInfoMsg(msg); + } + } else if (LOG_LW2M_VALUE.equals(typeMsg)) { + if (rpcRequest.getValueMsg() == null) { + rpcRequest.setValueMsg(msg); + } } this.onToDeviceRpcResponse(rpcRequest.getDeviceRpcResponseResultMsg(), rpcRequest.getSessionInfo()); } @@ -556,7 +578,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { */ protected void onAwakeDev(Registration registration) { log.info("[{}] [{}] Received endpoint Awake version event", registration.getId(), registration.getEndpoint()); - this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client is awake!", registration); + this.sendLogsToThingsboard(LOG_LW2M_INFO + ": Client is awake!", registration.getId()); //TODO: associate endpointId with device information. } @@ -583,10 +605,10 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { /** * @param logMsg - text msg - * @param registration - Id of Registration LwM2M Client + * @param registrationId - Id of Registration LwM2M Client */ - public void sendLogsToThingsboard(String logMsg, Registration registration) { - SessionInfoProto sessionInfo = this.getValidateSessionInfo(registration); + public void sendLogsToThingsboard(String logMsg, String registrationId) { + SessionInfoProto sessionInfo = this.getValidateSessionInfo(registrationId); if (logMsg != null && sessionInfo != null) { this.lwM2mTransportContextServer.sendParametersOnThingsboardTelemetry(this.lwM2mTransportContextServer.getKvLogyToThingsboard(logMsg), sessionInfo); } @@ -610,7 +632,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { if (clientObjects != null && clientObjects.size() > 0) { if (LWM2M_STRATEGY_2 == LwM2mTransportHandler.getClientOnlyObserveAfterConnect(lwM2MClientProfile)) { // #2 - lwM2MClient.getPendingRequests().addAll(clientObjects); + lwM2MClient.getPendingReadRequests().addAll(clientObjects); clientObjects.forEach(path -> lwM2mTransportRequest.sendAllRequest(registration, path, READ, ContentFormat.TLV.getName(), null, this.lwM2mTransportContextServer.getLwM2MTransportConfigServer().getTimeout(), null)); } @@ -652,6 +674,8 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { * Sending observe value of resources to thingsboard * #1 Return old Value Resource from LwM2MClient * #2 Update new Resources (replace old Resource Value on new Resource Value) + * #3 If fr_update -> UpdateFirmware + * #4 updateAttrTelemetry * * @param registration - Registration LwM2M Client * @param lwM2mResource - LwM2mSingleResource response.getContent() @@ -661,6 +685,19 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { LwM2mClient lwM2MClient = lwM2mClientContext.getLwM2mClientWithReg(registration, null); if (lwM2MClient.saveResourceValue(path, lwM2mResource, this.lwM2mTransportContextServer.getLwM2MTransportConfigServer() .getModelProvider())) { + if (FR_PATH_RESOURCE_VER_ID.equals(convertPathFromIdVerToObjectId(path)) && + lwM2MClient.getFrUpdate().getCurrentFwVersion() != null + && !lwM2MClient.getFrUpdate().getCurrentFwVersion().equals(lwM2MClient.getFrUpdate().getClientFwVersion()) + && lwM2MClient.isUpdateFw()) { + + /** version != null + * set setClient_fw_version = value + **/ + lwM2MClient.setUpdateFw(false); + lwM2MClient.getFrUpdate().setClientFwVersion(lwM2mResource.getValue().toString()); + log.warn("updateFirmwareClient3"); + this.updateFirmwareClient(lwM2MClient); + } Set paths = new HashSet<>(); paths.add(path); this.updateAttrTelemetry(registration, paths); @@ -669,6 +706,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { } } + /** * send Attribute and Telemetry to Thingsboard * #1 - get AttrName/TelemetryName with value from LwM2MClient: @@ -723,22 +761,23 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { params = this.getPathForWriteAttributes(lwM2MClientProfile.getPostAttributeLwm2mProfile()); result = params.keySet(); } - if (!result.isEmpty()) { + if (result != null && !result.isEmpty()) { // #1 Set pathSend = result.stream().filter(target -> { return target.split(LWM2M_SEPARATOR_PATH).length < 3 ? clientObjects.contains("/" + target.split(LWM2M_SEPARATOR_PATH)[1]) : clientObjects.contains("/" + target.split(LWM2M_SEPARATOR_PATH)[1] + "/" + target.split(LWM2M_SEPARATOR_PATH)[2]); } - ) - .collect(Collectors.toUnmodifiableSet()); + ).collect(Collectors.toUnmodifiableSet()); if (!pathSend.isEmpty()) { - lwM2MClient.getPendingRequests().addAll(pathSend); + lwM2MClient.getPendingReadRequests().addAll(pathSend); ConcurrentHashMap finalParams = params; - pathSend.forEach(target -> lwM2mTransportRequest.sendAllRequest(registration, target, typeOper, ContentFormat.TLV.getName(), - finalParams != null ? finalParams.get(target) : null, this.lwM2mTransportContextServer.getLwM2MTransportConfigServer().getTimeout(), null)); + pathSend.forEach(target -> { + lwM2mTransportRequest.sendAllRequest(registration, target, typeOper, ContentFormat.TLV.getName(), + finalParams != null ? finalParams.get(target) : null, this.lwM2mTransportContextServer.getLwM2MTransportConfigServer().getTimeout(), null); + }); if (OBSERVE.equals(typeOper)) { - lwM2MClient.initValue(this, null); + lwM2MClient.initReadValue(this, null); } } } @@ -969,7 +1008,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { // update value in Resources registrationIds.forEach(registrationId -> { Registration registration = lwM2mClientContext.getRegistration(registrationId); - this.readResourceValueObserve(registration, sendAttrToThingsboard.getPathPostParametersAdd(), READ); + this.readObserveFromProfile(registration, sendAttrToThingsboard.getPathPostParametersAdd(), READ); // send attr/telemetry to tingsboard for new path this.updateAttrTelemetry(registration, sendAttrToThingsboard.getPathPostParametersAdd()); }); @@ -998,12 +1037,12 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { registrationIds.forEach(registrationId -> { Registration registration = lwM2mClientContext.getRegistration(registrationId); if (postObserveAnalyzer.getPathPostParametersAdd().size() > 0) { - this.readResourceValueObserve(registration, postObserveAnalyzer.getPathPostParametersAdd(), OBSERVE); + this.readObserveFromProfile(registration, postObserveAnalyzer.getPathPostParametersAdd(), OBSERVE); } // 5.3 del // send Request cancel observe to Client if (postObserveAnalyzer.getPathPostParametersDel().size() > 0) { - this.cancelObserveIsValue(registration, postObserveAnalyzer.getPathPostParametersDel()); + this.cancelObserveFromProfile(registration, postObserveAnalyzer.getPathPostParametersDel()); } }); } @@ -1043,7 +1082,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { * @param registration - Registration LwM2M Client * @param targets - path Resources == [ "/2/0/0", "/2/0/1"] */ - private void readResourceValueObserve(Registration registration, Set targets, LwM2mTypeOper typeOper) { + private void readObserveFromProfile(Registration registration, Set targets, LwM2mTypeOper typeOper) { targets.forEach(target -> { LwM2mPath pathIds = new LwM2mPath(convertPathFromIdVerToObjectId(target)); if (pathIds.isResource()) { @@ -1133,7 +1172,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { } - private void cancelObserveIsValue(Registration registration, Set paramAnallyzer) { + private void cancelObserveFromProfile(Registration registration, Set paramAnallyzer) { LwM2mClient lwM2MClient = lwM2mClientContext.getLwM2mClientWithReg(registration, null); paramAnallyzer.forEach(pathIdVer -> { if (this.getResourceValueFromLwM2MClient(lwM2MClient, pathIdVer) != null) { @@ -1153,7 +1192,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { log.error("Failed update resource [{}] [{}]", path, valueNew); String logMsg = String.format("%s: Failed update resource path - %s value - %s. Value is not changed or bad", LOG_LW2M_ERROR, path, valueNew); - this.sendLogsToThingsboard(logMsg, lwM2MClient.getRegistration()); + this.sendLogsToThingsboard(logMsg, lwM2MClient.getRegistration().getId()); log.info("Failed update resource [{}] [{}]", path, valueNew); } } @@ -1182,8 +1221,10 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { } /** - * Update resource value on client: if there is a difference in values between the current resource values and the shared attribute values - * #1 Get path resource by result attributesResponse + * 1. FirmwareUpdate: + * - msg.getSharedUpdatedList().forEach(tsKvProto -> {tsKvProto.getKv().getKey().indexOf(FIRMWARE_UPDATE_PREFIX, 0) == 0 + * 2. Update resource value on client: if there is a difference in values between the current resource values and the shared attribute values + * - Get path resource by result attributesResponse * * @param attributesResponse - * @param sessionInfo - @@ -1191,6 +1232,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { public void onGetAttributesResponse(TransportProtos.GetAttributeResponseMsg attributesResponse, TransportProtos.SessionInfoProto sessionInfo) { try { List tsKvProtos = attributesResponse.getSharedAttributeListList(); + this.updateAttriuteFromThingsboard(tsKvProtos, sessionInfo); } catch (Exception e) { log.error(String.valueOf(e)); @@ -1274,7 +1316,7 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { */ private SessionInfoProto getValidateSessionInfo(String registrationId) { LwM2mClient lwM2MClient = lwM2mClientContext.getLwM2mClientWithReg(null, registrationId); - return getNewSessionInfoProto(lwM2MClient); + return lwM2MClient != null ? this.getNewSessionInfoProto(lwM2MClient) : null; } /** @@ -1293,28 +1335,88 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { } /** - * !!! sharedAttr === profileAttr !!! - * If there is a difference in values between the current resource values and the shared attribute values - * when the client connects to the server - * #1 get attributes name from profile include name resources in ModelObject if resource isWritable - * #2.1 #1 size > 0 => send Request getAttributes to thingsboard + * #1. !!! sharedAttr === profileAttr !!! + * - If there is a difference in values between the current resource values and the shared attribute values + * - when the client connects to the server + * #1.1 get attributes name from profile include name resources in ModelObject if resource isWritable + * #1.2 #1 size > 0 => send Request getAttributes to thingsboard + * #2. FirmwareAttribute subscribe: * * @param lwM2MClient - LwM2M Client */ public void putDelayedUpdateResourcesThingsboard(LwM2mClient lwM2MClient) { SessionInfoProto sessionInfo = this.getValidateSessionInfo(lwM2MClient.getRegistration()); if (sessionInfo != null) { - //#1.1 + #1.2 - List attrSharedNames = this.getNamesAttrFromProfileIsWritable(lwM2MClient); - if (attrSharedNames.size() > 0) { - //#2.1 + //#1.1 + ConcurrentMap keyNamesMap = this.getNamesFromProfileForSharedAttributes(lwM2MClient); + if (keyNamesMap.values().size() > 0) { try { - TransportProtos.GetAttributeRequestMsg getAttributeMsg = lwM2mTransportContextServer.getAdaptor().convertToGetAttributes(null, attrSharedNames); + //#1.2 + TransportProtos.GetAttributeRequestMsg getAttributeMsg = lwM2mTransportContextServer.getAdaptor().convertToGetAttributes(null, keyNamesMap.values()); transportService.process(sessionInfo, getAttributeMsg, getAckCallback(lwM2MClient, getAttributeMsg.getRequestId(), DEVICE_ATTRIBUTES_REQUEST)); } catch (AdaptorException e) { log.warn("Failed to decode get attributes request", e); } } + + } + } + + public void getInfoFirmwareUpdate(LwM2mClient lwM2MClient) { + SessionInfoProto sessionInfo = this.getValidateSessionInfo(lwM2MClient.getRegistration()); + if (sessionInfo != null) { + TransportProtos.GetFirmwareRequestMsg getFirmwareRequestMsg = TransportProtos.GetFirmwareRequestMsg.newBuilder() + .setDeviceIdMSB(sessionInfo.getDeviceIdMSB()) + .setDeviceIdLSB(sessionInfo.getDeviceIdLSB()) + .setTenantIdMSB(sessionInfo.getTenantIdMSB()) + .setTenantIdLSB(sessionInfo.getTenantIdLSB()) + .build(); + transportService.process(sessionInfo, getFirmwareRequestMsg, + new TransportServiceCallback<>() { + @Override + public void onSuccess(TransportProtos.GetFirmwareResponseMsg response) { + if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus())) { + lwM2MClient.getFrUpdate().setCurrentFwVersion(response.getVersion()); + lwM2MClient.getFrUpdate().setCurrentFwId(new FirmwareId(new UUID(response.getFirmwareIdMSB(), response.getFirmwareIdLSB())).getId()); + lwM2MClient.setUpdateFw(true); + readRequestToClientFirmwareVer(lwM2MClient.getRegistration()); + } else { + log.trace("Firmware [{}] [{}]", lwM2MClient.getDeviceName(), response.getResponseStatus().toString()); + } + } + + @Override + public void onError(Throwable e) { + log.trace("Failed to process credentials ", e); + } + }); + } + } + + /** + * @param registration + */ + public void readRequestToClientFirmwareVer(Registration registration) { + String pathIdVer = convertPathFromObjectIdToIdVer(FR_PATH_RESOURCE_VER_ID, registration); + lwM2mTransportRequest.sendAllRequest(registration, pathIdVer, READ, ContentFormat.TLV.getName(), + null, lwM2mTransportContextServer.getLwM2MTransportConfigServer().getTimeout(), null); + } + + /** + * + * @param lwM2MClient - + */ + public void updateFirmwareClient(LwM2mClient lwM2MClient) { + if (!lwM2MClient.getFrUpdate().getCurrentFwVersion().equals(lwM2MClient.getFrUpdate().getClientFwVersion())) { + int chunkSize = 0; + int chunk = 0; + byte[] firmwareChunk = firmwareDataCache.get(lwM2MClient.getFrUpdate().getCurrentFwId().toString(), chunkSize, chunk); + Integer objectId = 5; + String verSupportedObject = lwM2MClient.getRegistration().getSupportedObject().get(objectId); + String targetIdVer = LWM2M_SEPARATOR_PATH + objectId + LWM2M_SEPARATOR_KEY + verSupportedObject + LWM2M_SEPARATOR_PATH + 0 + LWM2M_SEPARATOR_PATH + 0; + lwM2mTransportRequest.sendAllRequest(lwM2MClient.getRegistration(), targetIdVer, WRITE_REPLACE, ContentFormat.TLV.getName(), + firmwareChunk, lwM2mTransportContextServer.getLwM2MTransportConfigServer().getTimeout(), null); + log.warn("updateFirmwareClient [{}] [{}]", lwM2MClient.getFrUpdate().getCurrentFwVersion(), lwM2MClient.getFrUpdate().getClientFwVersion()); } } @@ -1326,23 +1428,12 @@ public class LwM2mTransportServiceImpl implements LwM2mTransportService { * @param lwM2MClient - * @return ArrayList keyNames from profile profileAttr && IsWritable */ - private List getNamesAttrFromProfileIsWritable(LwM2mClient lwM2MClient) { + private ConcurrentMap getNamesFromProfileForSharedAttributes(LwM2mClient lwM2MClient) { + LwM2mClientProfile profile = lwM2mClientContext.getProfile(lwM2MClient.getProfileId()); - Set attrSet = new Gson().fromJson(profile.getPostAttributeProfile(), - new TypeToken>() { - }.getType()); - ConcurrentMap keyNamesMap = new Gson().fromJson(profile.getPostKeyNameProfile().toString(), + return new Gson().fromJson(profile.getPostKeyNameProfile().toString(), new TypeToken>() { }.getType()); - - ConcurrentMap keyNamesIsWritable = keyNamesMap.entrySet() - .stream() - .filter(e -> (attrSet.contains(e.getKey()) && validateResourceInModel(lwM2MClient, e.getKey(), true))) - .collect(Collectors.toConcurrentMap(Map.Entry::getKey, Map.Entry::getValue)); - - Set namesIsWritable = ConcurrentHashMap.newKeySet(); - namesIsWritable.addAll(new HashSet<>(keyNamesIsWritable.values())); - return new ArrayList<>(namesIsWritable); } private boolean validateResourceInModel(LwM2mClient lwM2mClient, String pathIdVer, boolean isWritableNotOptional) { diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java index b228a52b94..65a1e14a4c 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MJsonAdaptor.java @@ -24,11 +24,8 @@ import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; +import java.util.Collection; import java.util.Random; -import java.util.Set; @Slf4j @Component("LwM2MJsonAdaptor") @@ -54,11 +51,7 @@ public class LwM2MJsonAdaptor implements LwM2MTransportAdaptor { } @Override - public TransportProtos.GetAttributeRequestMsg convertToGetAttributes(List clientKeys, List sharedKeys) throws AdaptorException { - return processGetAttributeRequestMsg(clientKeys, sharedKeys); - } - - protected TransportProtos.GetAttributeRequestMsg processGetAttributeRequestMsg(List clientKeys, List sharedKeys) throws AdaptorException { + public TransportProtos.GetAttributeRequestMsg convertToGetAttributes(Collection clientKeys, Collection sharedKeys) throws AdaptorException { try { TransportProtos.GetAttributeRequestMsg.Builder result = TransportProtos.GetAttributeRequestMsg.newBuilder(); Random random = new Random(); @@ -75,14 +68,4 @@ public class LwM2MJsonAdaptor implements LwM2MTransportAdaptor { throw new AdaptorException(e); } } - - private Set toStringSet(JsonElement requestBody, String name) { - JsonElement element = requestBody.getAsJsonObject().get(name); - if (element != null) { - return new HashSet<>(Arrays.asList(element.getAsString().split(","))); - } else { - return null; - } - } - } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java index 4b6d54c874..d654475f03 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/adaptors/LwM2MTransportAdaptor.java @@ -19,7 +19,7 @@ import com.google.gson.JsonElement; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos; -import java.util.List; +import java.util.Collection; public interface LwM2MTransportAdaptor { @@ -27,5 +27,5 @@ public interface LwM2MTransportAdaptor { TransportProtos.PostAttributeMsg convertToPostAttributes(JsonElement jsonElement) throws AdaptorException; - TransportProtos.GetAttributeRequestMsg convertToGetAttributes(List clientKeys, List sharedKeys) throws AdaptorException; + TransportProtos.GetAttributeRequestMsg convertToGetAttributes(Collection clientKeys, Collection sharedKeys) throws AdaptorException; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java index ac1738e101..0617847246 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClient.java @@ -57,13 +57,15 @@ public class LwM2mClient implements Cloneable { private UUID deviceId; private UUID sessionId; private UUID profileId; + private volatile LwM2mFirmwareUpdate frUpdate; private Registration registration; private ValidateDeviceCredentialsResponseMsg credentialsResponse; private final Map resources; private final Map delayedRequests; - private final List pendingRequests; + private final List pendingReadRequests; private final Queue queuedRequests; private boolean init; + private volatile boolean updateFw; public Object clone() throws CloneNotSupportedException { return super.clone(); @@ -75,12 +77,14 @@ public class LwM2mClient implements Cloneable { this.securityInfo = securityInfo; this.credentialsResponse = credentialsResponse; this.delayedRequests = new ConcurrentHashMap<>(); - this.pendingRequests = new CopyOnWriteArrayList<>(); + this.pendingReadRequests = new CopyOnWriteArrayList<>(); this.resources = new ConcurrentHashMap<>(); this.profileId = profileId; this.sessionId = sessionId; this.init = false; + this.updateFw = false; this.queuedRequests = new ConcurrentLinkedQueue<>(); + this.frUpdate = new LwM2mFirmwareUpdate(); } public boolean saveResourceValue(String pathRez, LwM2mResource rez, LwM2mModelProvider modelProvider) { @@ -103,15 +107,13 @@ public class LwM2mClient implements Cloneable { LwM2mPath pathIds = new LwM2mPath(convertPathFromIdVerToObjectId(pathRez)); String verSupportedObject = registration.getSupportedObject().get(pathIds.getObjectId()); String verRez = getVerFromPathIdVerOrId(pathRez); - return (verRez == null || verSupportedObject.equals(verRez)) ? modelProvider.getObjectModel(registration) + return verRez == null || verRez.equals(verSupportedObject) ? modelProvider.getObjectModel(registration) .getResourceModel(pathIds.getObjectId(), pathIds.getResourceId()) : null; } public Collection getNewResourcesForInstance(String pathRezIdVer, LwM2mModelProvider modelProvider, LwM2mValueConverterImpl converter) { LwM2mPath pathIds = new LwM2mPath(convertPathFromIdVerToObjectId(pathRezIdVer)); - String verSupportedObject = registration.getSupportedObject().get(pathIds.getObjectId()); - String verRez = getVerFromPathIdVerOrId(pathRezIdVer); Collection resources = ConcurrentHashMap.newKeySet(); Map resourceModels = modelProvider.getObjectModel(registration) .getObjectModel(pathIds.getObjectId()).resources; @@ -170,11 +172,11 @@ public class LwM2mClient implements Cloneable { .collect(Collectors.toSet()); } - public void initValue(LwM2mTransportServiceImpl serviceImpl, String path) { + public void initReadValue(LwM2mTransportServiceImpl serviceImpl, String path) { if (path != null) { - this.pendingRequests.remove(path); + this.pendingReadRequests.remove(path); } - if (this.pendingRequests.size() == 0) { + if (this.pendingReadRequests.size() == 0) { this.init = true; serviceImpl.putDelayedUpdateResourcesThingsboard(this); } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java index f496ed01b4..fe17ac3819 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mClientContextImpl.java @@ -82,12 +82,12 @@ public class LwM2mClientContextImpl implements LwM2mClientContext { @Override public LwM2mClient getLwM2mClientWithReg(Registration registration, String registrationId) { - LwM2mClient client = registrationId != null ? + LwM2mClient client = registrationId != null && this.lwM2mClients.containsKey(registrationId) ? this.lwM2mClients.get(registrationId) : - this.lwM2mClients.containsKey(registration.getId()) ? - this.lwM2mClients.get(registration.getId()) : - this.lwM2mClients.get(registration.getEndpoint()); - return client != null ? client : updateInSessionsLwM2MClient(registration); + registration !=null && this.lwM2mClients.containsKey(registration.getId()) ? + this.lwM2mClients.get(registration.getId()) : registration !=null && this.lwM2mClients.containsKey(registration) ? + this.lwM2mClients.get(registration.getEndpoint()) : null; + return client != null ? client : registration!= null ? updateInSessionsLwM2MClient(registration) : null; } @Override diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFirmwareUpdate.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFirmwareUpdate.java new file mode 100644 index 0000000000..9eb6321445 --- /dev/null +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/client/LwM2mFirmwareUpdate.java @@ -0,0 +1,27 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.client; + +import lombok.Data; + +import java.util.UUID; + +@Data +public class LwM2mFirmwareUpdate { + private volatile String clientFwVersion; + private volatile String currentFwVersion; + private volatile UUID currentFwId; +} diff --git a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts index 77a5956855..3d6b93b84c 100644 --- a/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts +++ b/ui-ngx/src/app/modules/home/components/profile/device/lwm2m/lwm2m-observe-attr-telemetry.component.ts @@ -24,6 +24,8 @@ import { NG_VALUE_ACCESSOR, Validators } from '@angular/forms'; +import { Store } from '@ngrx/store'; +import { AppState } from '@core/core.state'; import { coerceBooleanProperty } from '@angular/cdk/coercion'; import { ATTRIBUTE, @@ -38,6 +40,7 @@ import { } from './lwm2m-profile-config.models'; import { deepClone, isDefinedAndNotNull, isEqual, isUndefined } from '@core/utils'; import { MatDialog } from '@angular/material/dialog'; +import { TranslateService } from '@ngx-translate/core'; import { Lwm2mObjectAddInstancesData, Lwm2mObjectAddInstancesDialogComponent @@ -80,8 +83,10 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor @Input() disabled: boolean; - constructor(private fb: FormBuilder, - private dialog: MatDialog) { + constructor(private store: Store, + private fb: FormBuilder, + private dialog: MatDialog, + public translate: TranslateService) { this.observeAttrTelemetryFormGroup = this.fb.group({ [CLIENT_LWM2M]: this.fb.array([]) }); @@ -93,7 +98,7 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor } private propagateChange = (v: any) => { - } + }; registerOnChange(fn: any): void { this.propagateChange = fn; @@ -184,7 +189,7 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor this.observeAttrTelemetryFormGroup.get(CLIENT_LWM2M).updateValueAndValidity(); } - trackByParams = (index: number): number => { + trackByParams = (index: number, element: any): number => { return index; } @@ -312,7 +317,7 @@ export class Lwm2mObserveAttrTelemetryComponent implements ControlValueAccessor return objectName + ' <' + idVerObj + '>'; } getNameInstanceLwm2m = (instance: Instance, idVerObj: string): string => { - return ` instance <${idVerObj}/${instance.id}>`; + return ' instance <' + idVerObj + '/' + instance.id +'>'; } updateAttributeLwm2mObject = (event: Event, objectKeyId: number): void => { diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index bd7ec5e2a2..9c2253d0ab 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -1205,7 +1205,7 @@ "telemetry-label": "Telemetry", "key-name-label": "Key Name", "attribute-lwm2m-label": "AttrLwm2m", - "resource-tip": "ID & Original Name of the Resource", + "resource-tip": "ID & Original Name of the Resource (only Operations isReadable)", "is-observe-tip": "Is Observe", "not-observe-tip": "To observe select telemetry or attributes first", "is-attr-tip": "Is Attribute", From e6d3b1d3f69f9715bb26c04616a5142c3ee65192 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Thu, 29 Apr 2021 14:22:43 +0300 Subject: [PATCH 32/32] Fix for getting user method when additionalInfo is null and refactoring --- .../server/controller/UserController.java | 24 +++++++------------ 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index f254414902..9daacc3027 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -32,6 +32,7 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.User; @@ -89,13 +90,14 @@ public class UserController extends BaseController { try { UserId userId = new UserId(toUUID(strUserId)); User user = checkUserId(userId, Operation.READ); - if(!user.getAdditionalInfo().isNull()) { - processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), DEFAULT_DASHBOARD); - processDashboardIdFromAdditionalInfo((ObjectNode) user.getAdditionalInfo(), HOME_DASHBOARD); - } - UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId()); - if(userCredentials.isEnabled()) { - addUserCredentialsEnabled((ObjectNode) user.getAdditionalInfo()); + if(user.getAdditionalInfo().isObject()) { + ObjectNode additionalInfo = (ObjectNode) user.getAdditionalInfo(); + processDashboardIdFromAdditionalInfo(additionalInfo, DEFAULT_DASHBOARD); + processDashboardIdFromAdditionalInfo(additionalInfo, HOME_DASHBOARD); + UserCredentials userCredentials = userService.findUserCredentialsByUserId(user.getTenantId(), user.getId()); + if(userCredentials.isEnabled() && !additionalInfo.has("userCredentialsEnabled")) { + additionalInfo.put("userCredentialsEnabled", true); + } } return user; } catch (Exception e) { @@ -103,14 +105,6 @@ public class UserController extends BaseController { } } - private void addUserCredentialsEnabled(ObjectNode additionalInfo) { - if(!additionalInfo.isNull()) { - if(!additionalInfo.has("userCredentialsEnabled")) { - additionalInfo.put("userCredentialsEnabled", true); - } - } - } - @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @RequestMapping(value = "/user/tokenAccessEnabled", method = RequestMethod.GET) @ResponseBody