From 316a09854cb23918afe3610a941e86b00bfed547 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Sun, 12 Nov 2023 22:15:57 +0100 Subject: [PATCH 01/18] implemented device actor delete msg and fixed assign to tenant --- .../server/actors/device/DeviceActor.java | 3 ++ .../server/actors/tenant/TenantActor.java | 14 ++++++++ .../DefaultTbNotificationEntityService.java | 3 +- .../device/DefaultTbDeviceService.java | 2 ++ .../queue/DefaultTbClusterService.java | 15 +++++--- .../server/service/queue/ProtoUtils.java | 22 ++++++++++++ .../server/cluster/TbClusterService.java | 4 ++- common/cluster-api/src/main/proto/queue.proto | 8 +++++ .../server/common/msg/MsgType.java | 2 ++ .../msg/rule/engine/DeviceDeleteMsg.java | 36 +++++++++++++++++++ 10 files changed, 103 insertions(+), 6 deletions(-) create mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java index 48425d58db..d69b9aec0b 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActor.java @@ -64,6 +64,9 @@ public class DeviceActor extends ContextAwareActor { case DEVICE_ATTRIBUTES_UPDATE_TO_DEVICE_ACTOR_MSG: processor.processAttributesUpdate((DeviceAttributesEventNotificationMsg) msg); break; + case DEVICE_DELETE_TO_DEVICE_ACTOR_MSG: + ctx.stop(ctx.getSelf()); + break; case DEVICE_CREDENTIALS_UPDATE_TO_DEVICE_ACTOR_MSG: processor.processCredentialsUpdate(msg); break; diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index c9f5448df3..f508d967c9 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -53,10 +53,13 @@ import org.thingsboard.server.common.msg.queue.PartitionChangeMsg; import org.thingsboard.server.common.msg.queue.QueueToRuleEngineMsg; import org.thingsboard.server.common.msg.queue.RuleEngineException; import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.transport.msg.TransportToDeviceActorMsgWrapper; +import java.util.HashSet; import java.util.List; +import java.util.Set; @Slf4j public class TenantActor extends RuleChainManagerActor { @@ -65,8 +68,11 @@ public class TenantActor extends RuleChainManagerActor { private boolean isCore; private ApiUsageState apiUsageState; + private Set deletedDevices; + private TenantActor(ActorSystemContext systemContext, TenantId tenantId) { super(systemContext, tenantId); + this.deletedDevices = new HashSet<>(); } boolean cantFindTenant = false; @@ -221,6 +227,10 @@ public class TenantActor extends RuleChainManagerActor { if (!isCore) { log.warn("RECEIVED INVALID MESSAGE: {}", msg); } + if (deletedDevices.contains(msg.getDeviceId())) { + log.warn("RECEIVED MESSAGE FOR DELETED DEVICE: {}", msg); + return; + } TbActorRef deviceActor = getOrCreateDeviceActor(msg.getDeviceId()); if (priority) { deviceActor.tellWithHighPriority(msg); @@ -249,6 +259,10 @@ public class TenantActor extends RuleChainManagerActor { Edge edge = systemContext.getEdgeService().findEdgeById(tenantId, edgeId); edgeRpcService.updateEdge(tenantId, edge); } + } else if (msg.getEntityId().getEntityType() == EntityType.DEVICE && ComponentLifecycleEvent.DELETED == msg.getEvent()) { + DeviceId deviceId = (DeviceId) msg.getEntityId(); + onToDeviceActorMsg(new DeviceDeleteMsg(tenantId, deviceId), true); + deletedDevices.add(deviceId); } else if (isRuleEngine) { TbActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java index 72e8f44020..8682d18a62 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java @@ -112,7 +112,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS public void notifyDeleteDevice(TenantId tenantId, DeviceId deviceId, CustomerId customerId, Device device, User user, Object... additionalInfo) { gatewayNotificationsService.onDeviceDeleted(device); - tbClusterService.onDeviceDeleted(device, null); + tbClusterService.onDeviceDeleted(tenantId, device, null); logEntityAction(tenantId, deviceId, device, customerId, ActionType.DELETED, user, additionalInfo); } @@ -126,6 +126,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS @Override public void notifyAssignDeviceToTenant(TenantId tenantId, TenantId newTenantId, DeviceId deviceId, CustomerId customerId, Device device, Tenant tenant, User user, Object... additionalInfo) { + tbClusterService.onDeviceAssignedToTenant(tenantId, device); logEntityAction(tenantId, deviceId, device, customerId, ActionType.ASSIGNED_TO_TENANT, user, additionalInfo); pushAssignedFromNotification(tenant, newTenantId, device); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index ac2d656ba6..a3d04f6344 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -232,6 +232,8 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T Tenant tenant = tenantService.findTenantById(tenantId); Device assignedDevice = deviceService.assignDeviceToTenant(newTenantId, device); + + notificationEntityService.notifyAssignDeviceToTenant(tenantId, newTenantId, deviceId, assignedDevice.getCustomerId(), assignedDevice, tenant, user, newTenantId.toString(), newTenant.getName()); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 103d2b68e2..75a1baa5c6 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -308,10 +308,17 @@ public class DefaultTbClusterService implements TbClusterService { } @Override - public void onDeviceDeleted(Device device, TbQueueCallback callback) { - broadcastEntityDeleteToTransport(device.getTenantId(), device.getId(), device.getName(), callback); - sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), false, false, true); - broadcastEntityStateChangeEvent(device.getTenantId(), device.getId(), ComponentLifecycleEvent.DELETED); + public void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback) { + DeviceId deviceId = device.getId(); + broadcastEntityDeleteToTransport(tenantId, deviceId, device.getName(), callback); + sendDeviceStateServiceEvent(tenantId, deviceId, false, false, true); + broadcastEntityStateChangeEvent(tenantId, deviceId, ComponentLifecycleEvent.DELETED); + } + + @Override + public void onDeviceAssignedToTenant(TenantId oldTenantId, Device device) { + onDeviceDeleted(oldTenantId, device, null); + sendDeviceStateServiceEvent(device.getTenantId(), device.getId(), true, false, false); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java index 76a8e3e89a..8986ca8b51 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -44,6 +44,7 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg; import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; +import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; @@ -377,6 +378,21 @@ public class ProtoUtils { ); } + private static TransportProtos.DeviceDeletedMsgProto toProto(DeviceDeleteMsg msg) { + return TransportProtos.DeviceDeletedMsgProto.newBuilder() + .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) + .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) + .setDeviceIdMSB(msg.getDeviceId().getId().getMostSignificantBits()) + .setDeviceIdLSB(msg.getDeviceId().getId().getLeastSignificantBits()) + .build(); + } + + private static DeviceDeleteMsg fromProto(TransportProtos.DeviceDeletedMsgProto proto) { + return new DeviceDeleteMsg( + TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), + new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB()))); + } + public static TransportProtos.ToDeviceActorNotificationMsgProto toProto(ToDeviceActorNotificationMsg msg) { if (msg instanceof DeviceEdgeUpdateMsg) { DeviceEdgeUpdateMsg updateMsg = (DeviceEdgeUpdateMsg) msg; @@ -406,6 +422,10 @@ public class ProtoUtils { RemoveRpcActorMsg updateMsg = (RemoveRpcActorMsg) msg; TransportProtos.RemoveRpcActorMsgProto proto = toProto(updateMsg); return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setRemoveRpcActorMsg(proto).build(); + } else if (msg instanceof DeviceDeleteMsg) { + DeviceDeleteMsg updateMsg = (DeviceDeleteMsg) msg; + TransportProtos.DeviceDeletedMsgProto proto = toProto(updateMsg); + return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setDeviceAssignToTenantMsgProto(proto).build(); } return null; } @@ -425,6 +445,8 @@ public class ProtoUtils { return fromProto(proto.getFromDeviceRpcResponseMsg()); } else if (proto.hasRemoveRpcActorMsg()) { return fromProto(proto.getRemoveRpcActorMsg()); + } else if (proto.hasDeviceAssignToTenantMsgProto()) { + return fromProto(proto.getDeviceAssignToTenantMsgProto()); } return null; } diff --git a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java index 5b1eb1ae4f..5c3b9076ff 100644 --- a/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java +++ b/common/cluster-api/src/main/java/org/thingsboard/server/cluster/TbClusterService.java @@ -83,7 +83,9 @@ public interface TbClusterService extends TbQueueClusterService { void onDeviceUpdated(Device device, Device old); - void onDeviceDeleted(Device device, TbQueueCallback callback); + void onDeviceDeleted(TenantId tenantId, Device device, TbQueueCallback callback); + + void onDeviceAssignedToTenant(TenantId oldTenantId, Device device); void onResourceChange(TbResource resource, TbQueueCallback callback); diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 6116411b2e..5000e3025f 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -1004,6 +1004,13 @@ message RemoveRpcActorMsgProto { int64 deviceIdLSB = 6; } +message DeviceDeletedMsgProto { + int64 tenantIdMSB = 1; + int64 tenantIdLSB = 2; + int64 deviceIdMSB = 3; + int64 deviceIdLSB = 4; +} + message ToDeviceActorNotificationMsgProto { DeviceEdgeUpdateMsgProto deviceEdgeUpdateMsg = 1; DeviceNameOrTypeUpdateMsgProto deviceNameOrTypeMsg = 2; @@ -1012,6 +1019,7 @@ message ToDeviceActorNotificationMsgProto { ToDeviceRpcRequestActorMsgProto toDeviceRpcRequestMsg = 5; FromDeviceRpcResponseActorMsgProto fromDeviceRpcResponseMsg = 6; RemoveRpcActorMsgProto removeRpcActorMsg = 7; + DeviceDeletedMsgProto deviceAssignToTenantMsgProto = 8; } /** diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java index 0a39d8d51b..c3d3623d8e 100644 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/MsgType.java @@ -93,6 +93,8 @@ public enum MsgType { DEVICE_NAME_OR_TYPE_UPDATE_TO_DEVICE_ACTOR_MSG, + DEVICE_DELETE_TO_DEVICE_ACTOR_MSG, + DEVICE_EDGE_UPDATE_TO_DEVICE_ACTOR_MSG, DEVICE_RPC_REQUEST_TO_DEVICE_ACTOR_MSG, diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java new file mode 100644 index 0000000000..1bdab7409b --- /dev/null +++ b/common/message/src/main/java/org/thingsboard/server/common/msg/rule/engine/DeviceDeleteMsg.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.msg.rule.engine; + +import lombok.Data; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.msg.MsgType; +import org.thingsboard.server.common.msg.ToDeviceActorNotificationMsg; + +@Data +public class DeviceDeleteMsg implements ToDeviceActorNotificationMsg { + + private static final long serialVersionUID = 4679029228395462172L; + + private final TenantId tenantId; + private final DeviceId deviceId; + + @Override + public MsgType getMsgType() { + return MsgType.DEVICE_DELETE_TO_DEVICE_ACTOR_MSG; + } +} From deb19d24c0b4ebdcf8fdf07c5e94d6397fb413ab Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Nov 2023 10:57:27 +0100 Subject: [PATCH 02/18] added verification for the corresponding test --- .../controller/DeviceControllerTest.java | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index 778a11a552..b6afaa34e1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -51,7 +51,6 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.alarm.AlarmSeverity; -import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; @@ -73,7 +72,9 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; +import org.thingsboard.server.service.state.DeviceStateService; import java.util.ArrayList; import java.util.List; @@ -82,6 +83,9 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -106,6 +110,9 @@ public class DeviceControllerTest extends AbstractControllerTest { @SpyBean private GatewayNotificationsService gatewayNotificationsService; + @SpyBean + private DeviceStateService deviceStateService; + @Autowired private DeviceDao deviceDao; @@ -1340,6 +1347,28 @@ public class DeviceControllerTest extends AbstractControllerTest { ActionType.ASSIGNED_TO_TENANT, savedDifferentTenant.getId().getId().toString(), savedDifferentTenant.getTitle()); testNotificationUpdateGatewayNever(); + Mockito.verify(deviceStateService, times(1)).onQueueMsg( + argThat(proto -> + proto.getTenantIdMSB() == savedTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getDeleted() + ), + any() + ); + + Mockito.verify(deviceStateService, times(1)).onQueueMsg( + argThat(proto -> + proto.getTenantIdMSB() == savedDifferentTenant.getUuidId().getMostSignificantBits() && + proto.getTenantIdLSB() == savedDifferentTenant.getUuidId().getLeastSignificantBits() && + proto.getDeviceIdMSB() == savedDevice.getUuidId().getMostSignificantBits() && + proto.getDeviceIdLSB() == savedDevice.getUuidId().getLeastSignificantBits() && + proto.getAdded() + ), + any() + ); + login("tenant9@thingsboard.org", "testPassword1"); Device foundDevice1 = doGet("/api/device/" + assignedDevice.getId().getId(), Device.class); From 5de58644e550618a36d1ebec80995c1d32f36a4d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 13 Nov 2023 11:32:04 +0100 Subject: [PATCH 03/18] refactoring --- .../server/service/entitiy/device/DefaultTbDeviceService.java | 2 -- .../org/thingsboard/server/controller/DeviceControllerTest.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index a3d04f6344..ac2d656ba6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -232,8 +232,6 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T Tenant tenant = tenantService.findTenantById(tenantId); Device assignedDevice = deviceService.assignDeviceToTenant(newTenantId, device); - - notificationEntityService.notifyAssignDeviceToTenant(tenantId, newTenantId, deviceId, assignedDevice.getCustomerId(), assignedDevice, tenant, user, newTenantId.toString(), newTenant.getName()); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java index b6afaa34e1..cefe24e935 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceControllerTest.java @@ -72,7 +72,6 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.service.gateway_device.GatewayNotificationsService; import org.thingsboard.server.service.state.DeviceStateService; @@ -85,7 +84,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; From 80fc2d9b862f09b900fb3e0d2b7ce71ba98759a7 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 16 Nov 2023 15:06:00 +0100 Subject: [PATCH 04/18] improvements --- .../server/actors/tenant/TenantActor.java | 11 +++++++---- .../server/service/queue/ProtoUtils.java | 16 ++++++++-------- common/cluster-api/src/main/proto/queue.proto | 4 ++-- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java index f508d967c9..86f338e89d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java +++ b/application/src/main/java/org/thingsboard/server/actors/tenant/TenantActor.java @@ -228,7 +228,7 @@ public class TenantActor extends RuleChainManagerActor { log.warn("RECEIVED INVALID MESSAGE: {}", msg); } if (deletedDevices.contains(msg.getDeviceId())) { - log.warn("RECEIVED MESSAGE FOR DELETED DEVICE: {}", msg); + log.debug("RECEIVED MESSAGE FOR DELETED DEVICE: {}", msg); return; } TbActorRef deviceActor = getOrCreateDeviceActor(msg.getDeviceId()); @@ -250,7 +250,8 @@ public class TenantActor extends RuleChainManagerActor { log.info("[{}] Received API state update. Going to ENABLE Rule Engine execution.", tenantId); initRuleChains(); } - } else if (msg.getEntityId().getEntityType() == EntityType.EDGE) { + } + if (msg.getEntityId().getEntityType() == EntityType.EDGE) { EdgeId edgeId = new EdgeId(msg.getEntityId().getId()); EdgeRpcService edgeRpcService = systemContext.getEdgeRpcService(); if (msg.getEvent() == ComponentLifecycleEvent.DELETED) { @@ -259,11 +260,13 @@ public class TenantActor extends RuleChainManagerActor { Edge edge = systemContext.getEdgeService().findEdgeById(tenantId, edgeId); edgeRpcService.updateEdge(tenantId, edge); } - } else if (msg.getEntityId().getEntityType() == EntityType.DEVICE && ComponentLifecycleEvent.DELETED == msg.getEvent()) { + } + if (msg.getEntityId().getEntityType() == EntityType.DEVICE && ComponentLifecycleEvent.DELETED == msg.getEvent()) { DeviceId deviceId = (DeviceId) msg.getEntityId(); onToDeviceActorMsg(new DeviceDeleteMsg(tenantId, deviceId), true); deletedDevices.add(deviceId); - } else if (isRuleEngine) { + } + if (isRuleEngine) { TbActorRef target = getEntityActorRef(msg.getEntityId()); if (target != null) { if (msg.getEntityId().getEntityType() == EntityType.RULE_CHAIN) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java index 8986ca8b51..c93af1bb07 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/ProtoUtils.java @@ -44,9 +44,9 @@ import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponseActorMsg; import org.thingsboard.server.common.msg.rpc.RemoveRpcActorMsg; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequest; import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; -import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceAttributesEventNotificationMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceCredentialsUpdateNotificationMsg; +import org.thingsboard.server.common.msg.rule.engine.DeviceDeleteMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceEdgeUpdateMsg; import org.thingsboard.server.common.msg.rule.engine.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.gen.transport.TransportProtos; @@ -378,8 +378,8 @@ public class ProtoUtils { ); } - private static TransportProtos.DeviceDeletedMsgProto toProto(DeviceDeleteMsg msg) { - return TransportProtos.DeviceDeletedMsgProto.newBuilder() + private static TransportProtos.DeviceDeleteMsgProto toProto(DeviceDeleteMsg msg) { + return TransportProtos.DeviceDeleteMsgProto.newBuilder() .setTenantIdMSB(msg.getTenantId().getId().getMostSignificantBits()) .setTenantIdLSB(msg.getTenantId().getId().getLeastSignificantBits()) .setDeviceIdMSB(msg.getDeviceId().getId().getMostSignificantBits()) @@ -387,7 +387,7 @@ public class ProtoUtils { .build(); } - private static DeviceDeleteMsg fromProto(TransportProtos.DeviceDeletedMsgProto proto) { + private static DeviceDeleteMsg fromProto(TransportProtos.DeviceDeleteMsgProto proto) { return new DeviceDeleteMsg( TenantId.fromUUID(new UUID(proto.getTenantIdMSB(), proto.getTenantIdLSB())), new DeviceId(new UUID(proto.getDeviceIdMSB(), proto.getDeviceIdLSB()))); @@ -424,8 +424,8 @@ public class ProtoUtils { return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setRemoveRpcActorMsg(proto).build(); } else if (msg instanceof DeviceDeleteMsg) { DeviceDeleteMsg updateMsg = (DeviceDeleteMsg) msg; - TransportProtos.DeviceDeletedMsgProto proto = toProto(updateMsg); - return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setDeviceAssignToTenantMsgProto(proto).build(); + TransportProtos.DeviceDeleteMsgProto proto = toProto(updateMsg); + return TransportProtos.ToDeviceActorNotificationMsgProto.newBuilder().setDeviceDeleteMsg(proto).build(); } return null; } @@ -445,8 +445,8 @@ public class ProtoUtils { return fromProto(proto.getFromDeviceRpcResponseMsg()); } else if (proto.hasRemoveRpcActorMsg()) { return fromProto(proto.getRemoveRpcActorMsg()); - } else if (proto.hasDeviceAssignToTenantMsgProto()) { - return fromProto(proto.getDeviceAssignToTenantMsgProto()); + } else if (proto.hasDeviceDeleteMsg()) { + return fromProto(proto.getDeviceDeleteMsg()); } return null; } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 5000e3025f..c8919cb5c7 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -1004,7 +1004,7 @@ message RemoveRpcActorMsgProto { int64 deviceIdLSB = 6; } -message DeviceDeletedMsgProto { +message DeviceDeleteMsgProto { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; int64 deviceIdMSB = 3; @@ -1019,7 +1019,7 @@ message ToDeviceActorNotificationMsgProto { ToDeviceRpcRequestActorMsgProto toDeviceRpcRequestMsg = 5; FromDeviceRpcResponseActorMsgProto fromDeviceRpcResponseMsg = 6; RemoveRpcActorMsgProto removeRpcActorMsg = 7; - DeviceDeletedMsgProto deviceAssignToTenantMsgProto = 8; + DeviceDeleteMsgProto deviceDeleteMsg = 8; } /** From 336161e1a2101ad442570021699fa15c73319089 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Fri, 17 Nov 2023 17:36:22 +0200 Subject: [PATCH 05/18] Added fix for docker command for gateway launch command and support for ipv6 adresses in device connectivity --- .../DeviceConnectivityController.java | 2 +- .../DeviceConnectivityControllerTest.java | 271 ++++++++++++------ .../device/DeviceConnectivityServiceImpl.java | 20 +- .../dao/util/DeviceConnectivityUtil.java | 50 +++- 4 files changed, 235 insertions(+), 108 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java index 83d4d16034..dca113621c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceConnectivityController.java @@ -106,7 +106,7 @@ public class DeviceConnectivityController extends BaseController { @RequestMapping(value = "/device-connectivity/gateway-launch/{deviceId}", method = RequestMethod.GET) @ResponseBody public JsonNode getGatewayLaunchCommands(@ApiParam(value = DEVICE_ID_PARAM_DESCRIPTION) - @PathVariable(DEVICE_ID) String strDeviceId, HttpServletRequest request) throws ThingsboardException, URISyntaxException { + @PathVariable(DEVICE_ID) String strDeviceId, HttpServletRequest request) throws ThingsboardException, URISyntaxException { checkParameter(DEVICE_ID, strDeviceId); DeviceId deviceId = new DeviceId(toUUID(strDeviceId)); Device device = checkDeviceId(deviceId, Operation.READ_CREDENTIALS); diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 227d37f483..b629408855 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -18,8 +18,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -32,7 +30,6 @@ import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -46,7 +43,6 @@ import org.thingsboard.server.common.data.device.profile.DefaultDeviceProfileCon import org.thingsboard.server.common.data.device.profile.DeviceProfileData; import org.thingsboard.server.common.data.device.profile.MqttDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.id.DeviceProfileId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; @@ -58,14 +54,16 @@ import java.nio.file.Path; import static org.assertj.core.api.Assertions.assertThat; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.CA_ROOT_CERT_PEM; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.COAP; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.COAPS; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.DOCKER; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTP; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTPS; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.LINUX; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTT; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTTS; -import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.CA_ROOT_CERT_PEM; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.WINDOWS; @TestPropertySource(properties = { "device.connectivity.mqtts.pem_cert_file=/tmp/" + CA_ROOT_CERT_PEM @@ -294,6 +292,140 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { DEVICE_TELEMETRY_TOPIC, credentials.getCredentialsId())); } + @Test + public void testFetchGatewayLaunchCommands() throws Exception { + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + ObjectNode additionalInfo = JacksonUtil.newObjectNode(); + additionalInfo.put("gateway", true); + device.setAdditionalInfo(additionalInfo); + Device savedDevice = doPost("/api/device", device, Device.class); + DeviceCredentials credentials = + doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class); + + JsonNode commands = + doGetTyped("/api/device-connectivity/gateway-launch/" + savedDevice.getId().getId(), new TypeReference<>() { + }); + + JsonNode dockerMqttCommands = commands.get(MQTT); + assertThat(dockerMqttCommands.get(LINUX).asText()).isEqualTo(String.format("docker run -it -v ~/.tb-gateway/logs:/thingsboard_gateway/logs -v ~/.tb-gateway/extensions:/thingsboard_gateway/extensions -v ~/.tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=1883 -e accessToken=%s --restart always thingsboard/tb-gateway", credentials.getCredentialsId())); + assertThat(dockerMqttCommands.get(WINDOWS).asText()).isEqualTo("docker run -it -v %HOMEPATH%/tb-gateway/logs:/thingsboard_gateway/logs -v %HOMEPATH%/tb-gateway/extensions:/thingsboard_gateway/extensions -v %HOMEPATH%/tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=1883 -e accessToken=" + credentials.getCredentialsId() + " --restart always thingsboard/tb-gateway"); + + JsonNode dockerMqttsCommands = commands.get(MQTTS); + assertThat(dockerMqttsCommands.get(LINUX).asText()).isEqualTo(String.format("docker run -it -v ~/.tb-gateway/logs:/thingsboard_gateway/logs -v ~/.tb-gateway/extensions:/thingsboard_gateway/extensions -v ~/.tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=8883 -e accessToken=%s --restart always thingsboard/tb-gateway", credentials.getCredentialsId())); + assertThat(dockerMqttsCommands.get(WINDOWS).asText()).isEqualTo("docker run -it -v %HOMEPATH%/tb-gateway/logs:/thingsboard_gateway/logs -v %HOMEPATH%/tb-gateway/extensions:/thingsboard_gateway/extensions -v %HOMEPATH%/tb-gateway/config:/thingsboard_gateway/config --network=host -p 5000:5000 --name tbGatewayLocalhost -e host=localhost -e port=8883 -e accessToken=" + credentials.getCredentialsId() + " --restart always thingsboard/tb-gateway"); + } + + @Test + public void testFetchPublishTelemetryCommandsForDeviceWithIpV6LocalhostAddress() throws Exception { + loginSysAdmin(); + setConnectivityHost("::1"); + loginTenantAdmin(); + + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + Device savedDevice = doPost("/api/device", device, Device.class); + DeviceCredentials credentials = + doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class); + + JsonNode commands = + doGetTyped("/api/device-connectivity/" + savedDevice.getId().getId(), new TypeReference<>() { + }); + + assertThat(commands).hasSize(3); + JsonNode httpCommands = commands.get(HTTP); + assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost:8080/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + + JsonNode mqttCommands = commands.get(MQTT); + assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h localhost -p 1883 -t v1/devices/me/telemetry " + + "-u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download"); + assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 " + + "-t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerMqttCommands = commands.get(MQTT).get(DOCKER); + assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h localhost" + + " -p 1883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients " + + "/bin/sh -c \"curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download && " + + "mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", + credentials.getCredentialsId())); + + JsonNode linuxCoapCommands = commands.get(COAP); + assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry " + + "-t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry" + + " -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER); + assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + + " thingsboard/coap-clients coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + + " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + } + + @Test + public void testFetchPublishTelemetryCommandsForDeviceWithIpV6Address() throws Exception { + + loginSysAdmin(); + setConnectivityHost("1:1:1:1:1:1:1:1"); + loginTenantAdmin(); + + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + Device savedDevice = doPost("/api/device", device, Device.class); + DeviceCredentials credentials = + doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class); + + JsonNode commands = + doGetTyped("/api/device-connectivity/" + savedDevice.getId().getId(), new TypeReference<>() { + }); + assertThat(commands).hasSize(3); + JsonNode httpCommands = commands.get(HTTP); + assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://[1:1:1:1:1:1:1:1]:8080/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://[1:1:1:1:1:1:1:1]/api/v1/%s/telemetry " + + "--header Content-Type:application/json --data \"{temperature:25}\"", + credentials.getCredentialsId())); + + JsonNode mqttCommands = commands.get(MQTT); + assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h 1:1:1:1:1:1:1:1 -p 1883 -t v1/devices/me/telemetry " + + "-u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download"); + assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h 1:1:1:1:1:1:1:1 -p 8883 " + + "-t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerMqttCommands = commands.get(MQTT).get(DOCKER); + assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h 1:1:1:1:1:1:1:1" + + " -p 1883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it thingsboard/mosquitto-clients " + + "/bin/sh -c \"curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download && " + + "mosquitto_pub -d -q 1 --cafile ca-root.pem -h 1:1:1:1:1:1:1:1 -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", + credentials.getCredentialsId())); + + JsonNode linuxCoapCommands = commands.get(COAP); + assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://[1:1:1:1:1:1:1:1]:5683/api/v1/%s/telemetry " + + "-t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://[1:1:1:1:1:1:1:1]:5684/api/v1/%s/telemetry" + + " -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + + JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER); + assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it" + + " thingsboard/coap-clients coap-client -v 6 -m POST coap://[1:1:1:1:1:1:1:1]:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it" + + " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://[1:1:1:1:1:1:1:1]:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + + } + @Test public void testFetchPublishTelemetryCommandsForDeviceWithMqttBasicCreds() throws Exception { Device device = new Device(); @@ -449,47 +581,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void testFetchPublishTelemetryCommandsForDefaultDeviceIfPortsSetToDefault() throws Exception { loginSysAdmin(); - ObjectNode config = JacksonUtil.newObjectNode(); - - ObjectNode http = JacksonUtil.newObjectNode(); - http.put("enabled", true); - http.put("host", ""); - http.put("port", 80); - config.set("http", http); - - ObjectNode https = JacksonUtil.newObjectNode(); - https.put("enabled", true); - https.put("host", ""); - https.put("port", 443); - config.set("https", https); - - ObjectNode mqtt = JacksonUtil.newObjectNode(); - mqtt.put("enabled", false); - mqtt.put("host", ""); - mqtt.put("port", 1883); - config.set("mqtt", mqtt); - - ObjectNode mqtts = JacksonUtil.newObjectNode(); - mqtts.put("enabled", false); - mqtts.put("host", ""); - mqtts.put("port", 8883); - config.set("mqtts", mqtts); - - ObjectNode coap = JacksonUtil.newObjectNode(); - coap.put("enabled", false); - coap.put("host", ""); - coap.put("port", 5683); - config.set("coap", coap); - - ObjectNode coaps = JacksonUtil.newObjectNode(); - coaps.put("enabled", false); - coaps.put("host", ""); - coaps.put("port", 5684); - config.set("coaps", coaps); - - AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); - adminSettings.setJsonValue(config); - doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); + setConnectivityHost(""); login("tenant2@thingsboard.org", "testPassword1"); @@ -518,47 +610,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void testFetchPublishTelemetryCommandsForDefaultDeviceIfHostIsNotLocalhost() throws Exception { loginSysAdmin(); - ObjectNode config = JacksonUtil.newObjectNode(); - - ObjectNode http = JacksonUtil.newObjectNode(); - http.put("enabled", true); - http.put("host", "test.domain"); - http.put("port", 8080); - config.set("http", http); - - ObjectNode https = JacksonUtil.newObjectNode(); - https.put("enabled", true); - https.put("host", "test.domain"); - https.put("port", 443); - config.set("https", https); - - ObjectNode mqtt = JacksonUtil.newObjectNode(); - mqtt.put("enabled", true); - mqtt.put("host", "test.domain"); - mqtt.put("port", 1883); - config.set("mqtt", mqtt); - - ObjectNode mqtts = JacksonUtil.newObjectNode(); - mqtts.put("enabled", true); - mqtts.put("host", "test.domain"); - mqtts.put("port", 8883); - config.set("mqtts", mqtts); - - ObjectNode coap = JacksonUtil.newObjectNode(); - coap.put("enabled", true); - coap.put("host", "test.domain"); - coap.put("port", 5683); - config.set("coap", coap); - - ObjectNode coaps = JacksonUtil.newObjectNode(); - coaps.put("enabled", true); - coaps.put("host", "test.domain"); - coaps.put("port", 5684); - config.set("coaps", coaps); - - AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); - adminSettings.setJsonValue(config); - doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); + setConnectivityHost("test.domain"); login("tenant2@thingsboard.org", "testPassword1"); @@ -612,4 +664,49 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it " + "thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://test.domain:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); } + + + private void setConnectivityHost(String host) throws Exception { + ObjectNode config = JacksonUtil.newObjectNode(); + + ObjectNode http = JacksonUtil.newObjectNode(); + http.put("enabled", true); + http.put("host", host); + http.put("port", 8080); + config.set("http", http); + + ObjectNode https = JacksonUtil.newObjectNode(); + https.put("enabled", true); + https.put("host", host); + https.put("port", 443); + config.set("https", https); + + ObjectNode mqtt = JacksonUtil.newObjectNode(); + mqtt.put("enabled", true); + mqtt.put("host", host); + mqtt.put("port", 1883); + config.set("mqtt", mqtt); + + ObjectNode mqtts = JacksonUtil.newObjectNode(); + mqtts.put("enabled", true); + mqtts.put("host", host); + mqtts.put("port", 8883); + config.set("mqtts", mqtts); + + ObjectNode coap = JacksonUtil.newObjectNode(); + coap.put("enabled", true); + coap.put("host", host); + coap.put("port", 5683); + config.set("coap", coap); + + ObjectNode coaps = JacksonUtil.newObjectNode(); + coaps.put("enabled", true); + coaps.put("host", host); + coaps.put("port", 5684); + config.set("coaps", coaps); + + AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); + adminSettings.setJsonValue(config); + doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java index 9f67093529..58f2f352d7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java @@ -43,7 +43,6 @@ import org.thingsboard.server.dao.util.DeviceConnectivityUtil; import java.io.InputStream; import java.io.InputStreamReader; -import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -64,6 +63,7 @@ import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.LINUX; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTT; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTTS; import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.WINDOWS; +import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.getHost; @Service("DeviceConnectivityDaoService") @Slf4j @@ -230,7 +230,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService deviceCredentials.getCredentialsType() != DeviceCredentialsType.ACCESS_TOKEN) { return null; } - String hostName = getHost(baseUrl, properties); + String hostName = getHost(baseUrl, properties, protocol); String propertiesPort = properties.getPort(); String port = (propertiesPort.isEmpty() || HTTP_DEFAULT_PORT.equals(propertiesPort) || HTTPS_DEFAULT_PORT.equals(propertiesPort)) ? "" : ":" + propertiesPort; @@ -278,14 +278,14 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private String getMqttPublishCommand(String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(MQTT); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, MQTT); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); return DeviceConnectivityUtil.getMqttPublishCommand(MQTT, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials); } private List getMqttsPublishCommand(String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(MQTTS); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, MQTTS); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); String pubCommand = DeviceConnectivityUtil.getMqttPublishCommand(MQTTS, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials); @@ -301,7 +301,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private JsonNode getGatewayDockerCommands(String baseUrl, DeviceCredentials deviceCredentials, String mqttType) throws URISyntaxException { ObjectNode dockerLaunchCommands = JacksonUtil.newObjectNode(); DeviceConnectivityInfo properties = getConnectivity(mqttType); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, mqttType); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); Optional.ofNullable(DeviceConnectivityUtil.getGatewayLaunchCommand(LINUX, mqttHost, mqttPort, deviceCredentials)) .ifPresent(v -> dockerLaunchCommands.put(LINUX, v)); @@ -312,7 +312,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private String getDockerMqttPublishCommand(String protocol, String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(protocol); - String mqttHost = getHost(baseUrl, properties); + String mqttHost = getHost(baseUrl, properties, protocol); String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort(); return DeviceConnectivityUtil.getDockerMqttPublishCommand(protocol, baseUrl, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials); } @@ -352,20 +352,16 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private String getCoapPublishCommand(String protocol, String baseUrl, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(protocol); - String hostName = getHost(baseUrl, properties); + String hostName = getHost(baseUrl, properties, protocol); String port = properties.getPort().isEmpty() ? "" : ":" + properties.getPort(); return DeviceConnectivityUtil.getCoapPublishCommand(protocol, hostName, port, deviceCredentials); } private String getDockerCoapPublishCommand(String protocol, String baseUrl, DeviceCredentials deviceCredentials) throws URISyntaxException { DeviceConnectivityInfo properties = getConnectivity(protocol); - String host = getHost(baseUrl, properties); + String host = getHost(baseUrl, properties, protocol); String port = properties.getPort().isEmpty() ? "" : ":" + properties.getPort(); return DeviceConnectivityUtil.getDockerCoapPublishCommand(protocol, host, port, deviceCredentials); } - private String getHost(String baseUrl, DeviceConnectivityInfo properties) throws URISyntaxException { - return properties.getHost().isEmpty() ? new URI(baseUrl).getHost() : properties.getHost(); - } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index aa446c12b5..f905a9d288 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -19,9 +19,14 @@ import org.apache.commons.lang3.StringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.dao.device.DeviceConnectivityInfo; -import java.util.Arrays; -import java.util.List; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.UnknownHostException; +import java.util.regex.Pattern; public class DeviceConnectivityUtil { @@ -41,7 +46,7 @@ public class DeviceConnectivityUtil { public static final String GATEWAY_DOCKER_RUN = "docker run -it "; public static final String MQTT_IMAGE = "thingsboard/mosquitto-clients "; public static final String COAP_IMAGE = "thingsboard/coap-clients "; - public static final List LOCAL_HOSTS = Arrays.asList("localhost", "127.0.0.1"); + private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(http|https|ftp)://.*$"); public static String getHttpPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { return String.format("curl -v -X POST %s://%s%s/api/v1/%s/telemetry --header Content-Type:application/json --data " + JSON_EXAMPLE_PAYLOAD, @@ -71,7 +76,7 @@ public class DeviceConnectivityUtil { command.append(" -u \"").append(credentials.getUserName()).append("\""); } if (credentials.getPassword() != null) { - command.append(" -P \"").append(credentials.getPassword()).append("\"");; + command.append(" -P \"").append(credentials.getPassword()).append("\""); } } else { return null; @@ -90,17 +95,19 @@ public class DeviceConnectivityUtil { gatewayVolumePathPrefix = "%HOMEPATH%/tb-gateway"; } - String gatewayContainerName = "tbGateway" + StringUtils.capitalize(host.replace(".", "")); + String gatewayContainerName = "tbGateway" + StringUtils.capitalize(host.replaceAll("[^A-Za-z0-9]", "")); StringBuilder command = new StringBuilder(GATEWAY_DOCKER_RUN); command.append("-v {gatewayVolumePathPrefix}/logs:/thingsboard_gateway/logs".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); command.append(" -v {gatewayVolumePathPrefix}/extensions:/thingsboard_gateway/extensions".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); command.append(" -v {gatewayVolumePathPrefix}/config:/thingsboard_gateway/config".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append(isLocalhost(host) ? " --network=host" : ""); + command.append(" -p 5000:5000"); command.append(" --name ").append(gatewayContainerName); command.append(" -e host=").append(host); command.append(" -e port=").append(port); - switch(deviceCredentials.getCredentialsType()) { + switch (deviceCredentials.getCredentialsType()) { case ACCESS_TOKEN: command.append(" -e accessToken=").append(deviceCredentials.getCredentialsId()); break; @@ -139,7 +146,7 @@ public class DeviceConnectivityUtil { } StringBuilder mqttDockerCommand = new StringBuilder(); - mqttDockerCommand.append(DOCKER_RUN).append(LOCAL_HOSTS.contains(host) ? "--network=host ":"").append(MQTT_IMAGE); + mqttDockerCommand.append(DOCKER_RUN).append(isLocalhost(host) ? "--network=host " : "").append(MQTT_IMAGE); if (MQTTS.equals(protocol)) { mqttDockerCommand.append("/bin/sh -c \"") @@ -171,6 +178,33 @@ public class DeviceConnectivityUtil { public static String getDockerCoapPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { String coapCommand = getCoapPublishCommand(protocol, host, port, deviceCredentials); - return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (LOCAL_HOSTS.contains(host) ? "--network=host ":""), COAP_IMAGE, coapCommand) : null; + return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (isLocalhost(host) ? "--network=host " : ""), COAP_IMAGE, coapCommand) : null; + } + + public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { + String host = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); + if (VALID_URL_PATTERN.matcher(host).matches()) { + host = new URI(host).getHost(); + } else { + try { + InetAddress inetAddress = InetAddress.getByName(host); + host = inetAddress.getCanonicalHostName().replaceAll("[\\[\\]]", ""); + if (inetAddress instanceof Inet6Address && !inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { + host = "[" + host + "]"; + } + } catch (UnknownHostException e) { + return host; + } + } + return host; + } + + private static boolean isLocalhost(String host) { + try { + InetAddress inetAddress = InetAddress.getByName(host); + return inetAddress.isLoopbackAddress(); + } catch (UnknownHostException e) { + return false; + } } } From 9a14260bf1d42d83882d452606a2e6e86263bf29 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Fri, 17 Nov 2023 18:54:01 +0200 Subject: [PATCH 06/18] Host check refactoring --- .../DeviceConnectivityControllerTest.java | 42 ++++++++++++++++++- .../dao/util/DeviceConnectivityUtil.java | 21 ++++++---- 2 files changed, 53 insertions(+), 10 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index b629408855..4ca59dc843 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -581,7 +581,47 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { public void testFetchPublishTelemetryCommandsForDefaultDeviceIfPortsSetToDefault() throws Exception { loginSysAdmin(); - setConnectivityHost(""); + ObjectNode config = JacksonUtil.newObjectNode(); + + ObjectNode http = JacksonUtil.newObjectNode(); + http.put("enabled", true); + http.put("host", ""); + http.put("port", 80); + config.set("http", http); + + ObjectNode https = JacksonUtil.newObjectNode(); + https.put("enabled", true); + https.put("host", ""); + https.put("port", 443); + config.set("https", https); + + ObjectNode mqtt = JacksonUtil.newObjectNode(); + mqtt.put("enabled", false); + mqtt.put("host", ""); + mqtt.put("port", 1883); + config.set("mqtt", mqtt); + + ObjectNode mqtts = JacksonUtil.newObjectNode(); + mqtts.put("enabled", false); + mqtts.put("host", ""); + mqtts.put("port", 8883); + config.set("mqtts", mqtts); + + ObjectNode coap = JacksonUtil.newObjectNode(); + coap.put("enabled", false); + coap.put("host", ""); + coap.put("port", 5683); + config.set("coap", coap); + + ObjectNode coaps = JacksonUtil.newObjectNode(); + coaps.put("enabled", false); + coaps.put("host", ""); + coaps.put("port", 5684); + config.set("coaps", coaps); + + AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class); + adminSettings.setJsonValue(config); + doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); login("tenant2@thingsboard.org", "testPassword1"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index f905a9d288..d1eb434100 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -183,17 +183,20 @@ public class DeviceConnectivityUtil { public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { String host = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); + InetAddress inetAddress; if (VALID_URL_PATTERN.matcher(host).matches()) { host = new URI(host).getHost(); - } else { - try { - InetAddress inetAddress = InetAddress.getByName(host); - host = inetAddress.getCanonicalHostName().replaceAll("[\\[\\]]", ""); - if (inetAddress instanceof Inet6Address && !inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { - host = "[" + host + "]"; - } - } catch (UnknownHostException e) { - return host; + } + try { + inetAddress = InetAddress.getByName(host); + host = inetAddress.getHostName(); + } catch (UnknownHostException e) { + return host; + } + if (inetAddress instanceof Inet6Address) { + host = host.replaceAll("[\\[\\]]", ""); + if (!inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { + host = "[" + host + "]"; } } return host; From d196e69b23be90617c78826cdddb6fa2642484f1 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 09:11:52 +0200 Subject: [PATCH 07/18] Updated regex to check url --- .../org/thingsboard/server/dao/util/DeviceConnectivityUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index d1eb434100..151932af1a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -46,7 +46,7 @@ public class DeviceConnectivityUtil { public static final String GATEWAY_DOCKER_RUN = "docker run -it "; public static final String MQTT_IMAGE = "thingsboard/mosquitto-clients "; public static final String COAP_IMAGE = "thingsboard/coap-clients "; - private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(http|https|ftp)://.*$"); + private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); public static String getHttpPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { return String.format("curl -v -X POST %s://%s%s/api/v1/%s/telemetry --header Content-Type:application/json --data " + JSON_EXAMPLE_PAYLOAD, From 8777f1297240470aa238ce3231c88fc25212f860 Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 09:47:25 +0200 Subject: [PATCH 08/18] Added null safety if user set a bad ipv6 address --- .../server/dao/util/DeviceConnectivityUtil.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index 151932af1a..44ddb095aa 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -182,12 +182,17 @@ public class DeviceConnectivityUtil { } public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { - String host = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); + String initialHost = properties.getHost().isEmpty() ? baseUrl : properties.getHost(); InetAddress inetAddress; - if (VALID_URL_PATTERN.matcher(host).matches()) { - host = new URI(host).getHost(); + String host = null; + if (VALID_URL_PATTERN.matcher(initialHost).matches()) { + host = new URI(initialHost).getHost(); + } + if (host == null) { + host = initialHost; } try { + host = host.replaceAll("^https?://", ""); inetAddress = InetAddress.getByName(host); host = inetAddress.getHostName(); } catch (UnknownHostException e) { From 5e8858d5637ad1801761c2de5998379ed3c6ef8b Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 11:17:19 +0200 Subject: [PATCH 09/18] Updated host check logic and related test --- .../DeviceConnectivityControllerTest.java | 20 +++++++++---------- .../dao/util/DeviceConnectivityUtil.java | 3 +-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index 4ca59dc843..f517d2e9c5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -336,39 +336,39 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { assertThat(commands).hasSize(3); JsonNode httpCommands = commands.get(HTTP); - assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost:8080/api/v1/%s/telemetry " + + assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://[::1]:8080/api/v1/%s/telemetry " + "--header Content-Type:application/json --data \"{temperature:25}\"", credentials.getCredentialsId())); - assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost/api/v1/%s/telemetry " + + assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://[::1]/api/v1/%s/telemetry " + "--header Content-Type:application/json --data \"{temperature:25}\"", credentials.getCredentialsId())); JsonNode mqttCommands = commands.get(MQTT); - assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h localhost -p 1883 -t v1/devices/me/telemetry " + + assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h ::1 -p 1883 -t v1/devices/me/telemetry " + "-u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download"); - assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 " + + assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile ca-root.pem -h ::1 -p 8883 " + "-t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); JsonNode dockerMqttCommands = commands.get(MQTT).get(DOCKER); - assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h localhost" + + assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h ::1" + " -p 1883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId())); assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it --network=host thingsboard/mosquitto-clients " + "/bin/sh -c \"curl -f -S -o ca-root.pem http://localhost:80/api/device-connectivity/mqtts/certificate/download && " + - "mosquitto_pub -d -q 1 --cafile ca-root.pem -h localhost -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", + "mosquitto_pub -d -q 1 --cafile ca-root.pem -h ::1 -p 8883 -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", credentials.getCredentialsId())); JsonNode linuxCoapCommands = commands.get(COAP); - assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry " + + assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://[::1]:5683/api/v1/%s/telemetry " + "-t json -e \"{temperature:25}\"", credentials.getCredentialsId())); - assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry" + + assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://[::1]:5684/api/v1/%s/telemetry" + " -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER); assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + - " thingsboard/coap-clients coap-client -v 6 -m POST coap://localhost:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + " thingsboard/coap-clients coap-client -v 6 -m POST coap://[::1]:5683/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it --network=host" + - " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://localhost:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); + " thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://[::1]:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId())); } @Test diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index 44ddb095aa..efd75e305e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -194,13 +194,12 @@ public class DeviceConnectivityUtil { try { host = host.replaceAll("^https?://", ""); inetAddress = InetAddress.getByName(host); - host = inetAddress.getHostName(); } catch (UnknownHostException e) { return host; } if (inetAddress instanceof Inet6Address) { host = host.replaceAll("[\\[\\]]", ""); - if (!inetAddress.isLoopbackAddress() && !protocol.startsWith(MQTT)) { + if (!protocol.startsWith(MQTT)) { host = "[" + host + "]"; } } From 8a926ae239820181778314c071e8f605fd63dbbf Mon Sep 17 00:00:00 2001 From: imbeacon Date: Mon, 20 Nov 2023 17:25:19 +0200 Subject: [PATCH 10/18] Changes, due to comments --- .../dao/util/DeviceConnectivityUtil.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index efd75e305e..3434b47965 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -44,6 +44,8 @@ public class DeviceConnectivityUtil { public static final String JSON_EXAMPLE_PAYLOAD = "\"{temperature:25}\""; public static final String DOCKER_RUN = "docker run --rm -it "; public static final String GATEWAY_DOCKER_RUN = "docker run -it "; + + public static final String NETWORK_HOST_PARAM = "--network=host "; public static final String MQTT_IMAGE = "thingsboard/mosquitto-clients "; public static final String COAP_IMAGE = "thingsboard/coap-clients "; private final static Pattern VALID_URL_PATTERN = Pattern.compile("^(https?)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]"); @@ -98,14 +100,14 @@ public class DeviceConnectivityUtil { String gatewayContainerName = "tbGateway" + StringUtils.capitalize(host.replaceAll("[^A-Za-z0-9]", "")); StringBuilder command = new StringBuilder(GATEWAY_DOCKER_RUN); - command.append("-v {gatewayVolumePathPrefix}/logs:/thingsboard_gateway/logs".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); - command.append(" -v {gatewayVolumePathPrefix}/extensions:/thingsboard_gateway/extensions".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); - command.append(" -v {gatewayVolumePathPrefix}/config:/thingsboard_gateway/config".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); - command.append(isLocalhost(host) ? " --network=host" : ""); - command.append(" -p 5000:5000"); - command.append(" --name ").append(gatewayContainerName); - command.append(" -e host=").append(host); - command.append(" -e port=").append(port); + command.append("-v {gatewayVolumePathPrefix}/logs:/thingsboard_gateway/logs ".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append("-v {gatewayVolumePathPrefix}/extensions:/thingsboard_gateway/extensions ".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append("-v {gatewayVolumePathPrefix}/config:/thingsboard_gateway/config ".replace("{gatewayVolumePathPrefix}", gatewayVolumePathPrefix)); + command.append(isLocalhost(host) ? NETWORK_HOST_PARAM : ""); + command.append("-p 5000:5000 "); + command.append("--name ").append(gatewayContainerName).append(" "); + command.append("-e host=").append(host).append(" "); + command.append("-e port=").append(port); switch (deviceCredentials.getCredentialsType()) { case ACCESS_TOKEN: @@ -146,7 +148,7 @@ public class DeviceConnectivityUtil { } StringBuilder mqttDockerCommand = new StringBuilder(); - mqttDockerCommand.append(DOCKER_RUN).append(isLocalhost(host) ? "--network=host " : "").append(MQTT_IMAGE); + mqttDockerCommand.append(DOCKER_RUN).append(isLocalhost(host) ? NETWORK_HOST_PARAM : "").append(MQTT_IMAGE); if (MQTTS.equals(protocol)) { mqttDockerCommand.append("/bin/sh -c \"") @@ -178,7 +180,7 @@ public class DeviceConnectivityUtil { public static String getDockerCoapPublishCommand(String protocol, String host, String port, DeviceCredentials deviceCredentials) { String coapCommand = getCoapPublishCommand(protocol, host, port, deviceCredentials); - return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (isLocalhost(host) ? "--network=host " : ""), COAP_IMAGE, coapCommand) : null; + return coapCommand != null ? String.format("%s%s%s", DOCKER_RUN + (isLocalhost(host) ? NETWORK_HOST_PARAM : ""), COAP_IMAGE, coapCommand) : null; } public static String getHost(String baseUrl, DeviceConnectivityInfo properties, String protocol) throws URISyntaxException { @@ -199,7 +201,7 @@ public class DeviceConnectivityUtil { } if (inetAddress instanceof Inet6Address) { host = host.replaceAll("[\\[\\]]", ""); - if (!protocol.startsWith(MQTT)) { + if (!MQTT.equals(protocol) && !MQTTS.equals(protocol)) { host = "[" + host + "]"; } } From 248bcefa809f86ccf9aa6003419585f28c204df0 Mon Sep 17 00:00:00 2001 From: nick Date: Sat, 25 Nov 2023 19:39:58 +0200 Subject: [PATCH 11/18] stringToBytes: refactoring, Input parameter Object as HexString, DigString, Integer, Byte --- .../thingsboard/script/api/tbel/TbUtils.java | 18 +++++- .../script/api/tbel/TbUtilsTest.java | 57 +++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 31bd634221..07c2baee2d 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -182,7 +182,7 @@ public class TbUtils { return TbJson.parse(ctx, jsonStr); } - public static String bytesToString(List bytesList) { + public static String bytesToString(List bytesList) { byte[] bytes = bytesFromList(bytesList); return new String(bytes); } @@ -210,10 +210,22 @@ public class TbUtils { } } - private static byte[] bytesFromList(List bytesList) { + private static byte[] bytesFromList(List bytesList) { byte[] bytes = new byte[bytesList.size()]; for (int i = 0; i < bytesList.size(); i++) { - bytes[i] = bytesList.get(i); + if (bytesList.get(i) instanceof Integer) { + byte val = ((Integer) bytesList.get(i)).byteValue(); + if (((Integer) bytesList.get(i)).intValue() > 255 || ((Integer) bytesList.get(i)).intValue() < -128){ + throw new NumberFormatException("The value " + bytesList.get(i) + " could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255) !"); + } else { + bytes[i] = val; } + } else if (bytesList.get(i) instanceof String) { + bytes[i] = parseInt((String) bytesList.get(i)).byteValue(); + } else if (bytesList.get(i) instanceof Byte) { + bytes[i] = (byte) bytesList.get(i); + } else { + throw new NumberFormatException("Value \"" + bytesList.get(i) + " could not be converted to a byte. Must be format HexDecimal/String/Integer !"); + } } return bytes; } diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index d2900d6ec4..b80ac85ca7 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.math.BigInteger; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.List; import java.util.Random; @@ -385,6 +386,62 @@ public class TbUtilsTest { Assert.assertThrows(IllegalAccessException.class, () -> TbUtils.stringToBytes(ctx, ((ExecutionHashMap) finalInputJson).get("hello"), "UTF-8")); } + @Test + public void bytesFromList() { + byte[] arrayBytes = {(byte)0x00, (byte)0x08, (byte)0x10, (byte)0x1C, (byte)0xFF, (byte)0xFC, (byte)0xAD, (byte)0x88, (byte)0x75, (byte)0x74, (byte)0x8A, (byte)0x82}; + Object[] arrayMix = { "0x00", 8, "16", "0x1C", 255, 252, 173, 136, 117, 116, -118, "-126"}; + + String expected = new String(arrayBytes); + ArrayList listBytes = new ArrayList<>(arrayBytes.length); + for (Byte element : arrayBytes) { + listBytes.add(element); + } + Assert.assertEquals(expected, TbUtils.bytesToString(listBytes)); + + ArrayList listMix = new ArrayList<>(arrayMix.length); + for (Object element : arrayMix) { + listMix.add(element); + } + Assert.assertEquals(expected, TbUtils.bytesToString(listMix)); + } + + @Test + public void bytesFromList_Error() { + List listHex = new ArrayList<>(); + listHex.add("0xFG"); + try { + TbUtils.bytesToString(listHex); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("Failed radix: [16] for value: \"FG\"!")); + } + listHex.add(0, "1F"); + try { + TbUtils.bytesToString(listHex); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("Failed radix: [10] for value: \"1F\"!")); + } + + ArrayList listIntBytes = new ArrayList<>(); + listIntBytes.add(-129); + try { + TbUtils.bytesToString(listIntBytes); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value -129 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + } + + listIntBytes.add(0, 256); + try { + TbUtils.bytesToString(listIntBytes); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value 256 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + } + } + + private static List toList(byte[] data) { List result = new ArrayList<>(data.length); for (Byte b : data) { From baa684f72b59b9a584a4ed8e003c14d30480ba64 Mon Sep 17 00:00:00 2001 From: nick Date: Sun, 26 Nov 2023 21:58:29 +0200 Subject: [PATCH 12/18] stringToBytes: refactoring test --- .../test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index b80ac85ca7..3a3fe22000 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -389,7 +389,7 @@ public class TbUtilsTest { @Test public void bytesFromList() { byte[] arrayBytes = {(byte)0x00, (byte)0x08, (byte)0x10, (byte)0x1C, (byte)0xFF, (byte)0xFC, (byte)0xAD, (byte)0x88, (byte)0x75, (byte)0x74, (byte)0x8A, (byte)0x82}; - Object[] arrayMix = { "0x00", 8, "16", "0x1C", 255, 252, 173, 136, 117, 116, -118, "-126"}; + Object[] arrayMix = { "0x00", 8, "16", "0x1C", 255, (byte)0xFC, 173, 136, 117, 116, -118, "-126"}; String expected = new String(arrayBytes); ArrayList listBytes = new ArrayList<>(arrayBytes.length); From 3ab4c8d468b2986d7884cd12694921991832fb95 Mon Sep 17 00:00:00 2001 From: nick Date: Sun, 26 Nov 2023 22:28:50 +0200 Subject: [PATCH 13/18] stringToBytes: refactoring --- .../src/main/java/org/thingsboard/script/api/tbel/TbUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 07c2baee2d..8fde4cd332 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -187,7 +187,7 @@ public class TbUtils { return new String(bytes); } - public static String bytesToString(List bytesList, String charsetName) throws UnsupportedEncodingException { + public static String bytesToString(List bytesList, String charsetName) throws UnsupportedEncodingException { byte[] bytes = bytesFromList(bytesList); return new String(bytes, charsetName); } From ee8dd9f98767d4f4b575031ebe5a089a76a30e79 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 27 Nov 2023 18:23:05 +0200 Subject: [PATCH 14/18] stringToBytes: comments2 --- .../script/api/tbel/TbUtilsTest.java | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java index 3a3fe22000..7d5ae00dcf 100644 --- a/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java +++ b/common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbUtilsTest.java @@ -423,13 +423,33 @@ public class TbUtilsTest { Assert.assertTrue(e.getMessage().contains("Failed radix: [10] for value: \"1F\"!")); } + List listIntString = new ArrayList<>(); + listIntString.add("-129"); + try { + TbUtils.bytesToString(listIntString); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value '-129' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); + } + + listIntString.add(0, "256"); + try { + TbUtils.bytesToString(listIntString); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value '256' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); + } + ArrayList listIntBytes = new ArrayList<>(); listIntBytes.add(-129); try { TbUtils.bytesToString(listIntBytes); Assert.fail("Should throw NumberFormatException"); } catch (NumberFormatException e) { - Assert.assertTrue(e.getMessage().contains("The value -129 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + Assert.assertTrue(e.getMessage().contains("The value '-129' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); } listIntBytes.add(0, 256); @@ -437,7 +457,20 @@ public class TbUtilsTest { TbUtils.bytesToString(listIntBytes); Assert.fail("Should throw NumberFormatException"); } catch (NumberFormatException e) { - Assert.assertTrue(e.getMessage().contains("The value 256 could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255")); + Assert.assertTrue(e.getMessage().contains("The value '256' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!")); + } + + ArrayList listObjects = new ArrayList<>(); + ArrayList listStringObjects = new ArrayList<>(); + listStringObjects.add("0xFD"); + listObjects.add(listStringObjects); + try { + TbUtils.bytesToString(listObjects); + Assert.fail("Should throw NumberFormatException"); + } catch (NumberFormatException e) { + Assert.assertTrue(e.getMessage().contains("The value '[0xFD]' could not be correctly converted to a byte. " + + "Must be a HexDecimal/String/Integer/Byte format !")); } } From 3af8b1a33e108f5be735fda54cfa66ec014a5b69 Mon Sep 17 00:00:00 2001 From: nick Date: Mon, 27 Nov 2023 18:24:22 +0200 Subject: [PATCH 15/18] stringToBytes: comments2 --- .../thingsboard/script/api/tbel/TbUtils.java | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java index 8fde4cd332..4a3f2d306b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/TbUtils.java @@ -213,18 +213,16 @@ public class TbUtils { private static byte[] bytesFromList(List bytesList) { byte[] bytes = new byte[bytesList.size()]; for (int i = 0; i < bytesList.size(); i++) { - if (bytesList.get(i) instanceof Integer) { - byte val = ((Integer) bytesList.get(i)).byteValue(); - if (((Integer) bytesList.get(i)).intValue() > 255 || ((Integer) bytesList.get(i)).intValue() < -128){ - throw new NumberFormatException("The value " + bytesList.get(i) + " could not be converted to a byte. Integer to byte needs only 8-bits (min/max = -128/127 or 0/255) !"); - } else { - bytes[i] = val; } - } else if (bytesList.get(i) instanceof String) { - bytes[i] = parseInt((String) bytesList.get(i)).byteValue(); - } else if (bytesList.get(i) instanceof Byte) { - bytes[i] = (byte) bytesList.get(i); + Object objectVal = bytesList.get(i); + if (objectVal instanceof Integer) { + bytes[i] = isValidIntegerToByte((Integer) objectVal); + } else if (objectVal instanceof String) { + bytes[i] = isValidIntegerToByte(parseInt((String) objectVal)); + } else if (objectVal instanceof Byte) { + bytes[i] = (byte) objectVal; } else { - throw new NumberFormatException("Value \"" + bytesList.get(i) + " could not be converted to a byte. Must be format HexDecimal/String/Integer !"); + throw new NumberFormatException("The value '" + objectVal + "' could not be correctly converted to a byte. " + + "Must be a HexDecimal/String/Integer/Byte format !"); } } return bytes; @@ -655,7 +653,7 @@ public class TbUtils { } } - public static boolean isValidRadix(String value, int radix) { + private static boolean isValidRadix(String value, int radix) { for (int i = 0; i < value.length(); i++) { if (i == 0 && value.charAt(i) == '-') { if (value.length() == 1) @@ -669,4 +667,12 @@ public class TbUtils { return true; } + private static byte isValidIntegerToByte (Integer val) { + if (val > 255 || val.intValue() < -128) { + throw new NumberFormatException("The value '" + val + "' could not be correctly converted to a byte. " + + "Integer to byte conversion requires the use of only 8 bits (with a range of min/max = -128/255)!"); + } else { + return val.byteValue(); + } + } } From 8ac80476ad2f3768d4f3ecefadd401da607b70a1 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 27 Nov 2023 23:13:03 +0100 Subject: [PATCH 16/18] added encode/decode time stats and refactoring --- .../cache/RedisTbTransactionalCache.java | 4 ++++ .../server/common/data/FstStatsService.java | 4 ++++ .../queue/util/ProtoWithFSTService.java | 8 +++++++- .../common/stats/FstStatsServiceImpl.java | 20 +++++++++++++++++-- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index c2dd66dde0..01a18ad7ad 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -84,8 +84,10 @@ public abstract class RedisTbTransactionalCache clazz); + void recordEncodeTime(Class clazz, long startTime); + + void recordDecodeTime(Class clazz, long startTime); + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index e35c86e070..204502068c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -36,8 +36,12 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { @Override public Optional decode(byte[] byteArray) { try { + long startTime = System.nanoTime(); Optional optional = Optional.ofNullable(FSTUtils.decode(byteArray)); - optional.ifPresent(obj -> fstStatsService.incrementDecode(obj.getClass())); + optional.ifPresent(obj -> { + fstStatsService.recordDecodeTime(obj.getClass(), startTime); + fstStatsService.incrementDecode(obj.getClass()); + }); return optional; } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); @@ -48,7 +52,9 @@ public class ProtoWithFSTService implements DataDecodingEncodingService { @Override public byte[] encode(T msq) { + long startTime = System.nanoTime(); var bytes = FSTUtils.encode(msq); + fstStatsService.recordEncodeTime(msq.getClass(), startTime); fstStatsService.incrementEncode(msq.getClass()); return bytes; } diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java index 5d78b8c36a..b5cb25f2dc 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/FstStatsServiceImpl.java @@ -15,28 +15,44 @@ */ package org.thingsboard.server.common.stats; +import io.micrometer.core.instrument.Timer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.FstStatsService; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; @Service public class FstStatsServiceImpl implements FstStatsService { private final ConcurrentHashMap encodeCounters = new ConcurrentHashMap<>(); private final ConcurrentHashMap decodeCounters = new ConcurrentHashMap<>(); + private final ConcurrentHashMap encodeTimers = new ConcurrentHashMap<>(); + private final ConcurrentHashMap decodeTimer = new ConcurrentHashMap<>(); @Autowired private StatsFactory statsFactory; @Override public void incrementEncode(Class clazz) { - encodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fstEncode", key)).increment(); + encodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fst_encode", key)).increment(); } @Override public void incrementDecode(Class clazz) { - decodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fstDecode", key)).increment(); + decodeCounters.computeIfAbsent(clazz.getSimpleName(), key -> statsFactory.createStatsCounter("fst_decode", key)).increment(); + } + + @Override + public void recordEncodeTime(Class clazz, long startTime) { + encodeTimers.computeIfAbsent(clazz.getSimpleName(), + key -> statsFactory.createTimer("fst_encode_time", "statsName", key)).record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); + } + + @Override + public void recordDecodeTime(Class clazz, long startTime) { + decodeTimer.computeIfAbsent(clazz.getSimpleName(), + key -> statsFactory.createTimer("fst_decode_time", "statsName", key)).record(System.nanoTime() - startTime, TimeUnit.NANOSECONDS); } } From 94315607d4c6adb96b3c5917d14874495f4d1ce7 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 28 Nov 2023 13:17:12 +0200 Subject: [PATCH 17/18] Edge Requests Service - fetch only first level of relation from cloud to edge --- .../rpc/sync/DefaultEdgeRequestsService.java | 2 +- .../server/edge/RelationEdgeTest.java | 30 ++++++++++++------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index af9f72e819..cd23f19556 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -294,7 +294,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { private ListenableFuture> findRelationByQuery(TenantId tenantId, Edge edge, EntityId entityId, EntitySearchDirection direction) { EntityRelationsQuery query = new EntityRelationsQuery(); - query.setParameters(new RelationsSearchParameters(entityId, direction, -1, false)); + query.setParameters(new RelationsSearchParameters(entityId, direction, 1, false)); return relationService.findByQuery(tenantId, query); } diff --git a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java index 04e6f3a3fb..cf82a85a13 100644 --- a/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/RelationEdgeTest.java @@ -129,14 +129,24 @@ public class RelationEdgeTest extends AbstractEdgeTest { Device device = findDeviceByName("Edge Device 1"); Asset asset = findAssetByName("Edge Asset 1"); - EntityRelation relation = new EntityRelation(); - relation.setType("test"); - relation.setFrom(device.getId()); - relation.setTo(asset.getId()); - relation.setTypeGroup(RelationTypeGroup.COMMON); + EntityRelation deviceToAssetRelation = new EntityRelation(); + deviceToAssetRelation.setType("test"); + deviceToAssetRelation.setFrom(device.getId()); + deviceToAssetRelation.setTo(asset.getId()); + deviceToAssetRelation.setTypeGroup(RelationTypeGroup.COMMON); edgeImitator.expectMessageAmount(1); - doPost("/api/relation", relation); + doPost("/api/relation", deviceToAssetRelation); + Assert.assertTrue(edgeImitator.waitForMessages()); + + EntityRelation assetToTenantRelation = new EntityRelation(); + assetToTenantRelation.setType("test"); + assetToTenantRelation.setFrom(asset.getId()); + assetToTenantRelation.setTo(tenantId); + assetToTenantRelation.setTypeGroup(RelationTypeGroup.COMMON); + + edgeImitator.expectMessageAmount(1); + doPost("/api/relation", assetToTenantRelation); Assert.assertTrue(edgeImitator.waitForMessages()); UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); @@ -159,16 +169,16 @@ public class RelationEdgeTest extends AbstractEdgeTest { Assert.assertTrue(latestMessage instanceof RelationUpdateMsg); RelationUpdateMsg relationUpdateMsg = (RelationUpdateMsg) latestMessage; Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, relationUpdateMsg.getMsgType()); - Assert.assertEquals(relation.getType(), relationUpdateMsg.getType()); + Assert.assertEquals(deviceToAssetRelation.getType(), relationUpdateMsg.getType()); UUID fromUUID = new UUID(relationUpdateMsg.getFromIdMSB(), relationUpdateMsg.getFromIdLSB()); EntityId fromEntityId = EntityIdFactory.getByTypeAndUuid(relationUpdateMsg.getFromEntityType(), fromUUID); - Assert.assertEquals(relation.getFrom(), fromEntityId); + Assert.assertEquals(deviceToAssetRelation.getFrom(), fromEntityId); UUID toUUID = new UUID(relationUpdateMsg.getToIdMSB(), relationUpdateMsg.getToIdLSB()); EntityId toEntityId = EntityIdFactory.getByTypeAndUuid(relationUpdateMsg.getToEntityType(), toUUID); - Assert.assertEquals(relation.getTo(), toEntityId); + Assert.assertEquals(deviceToAssetRelation.getTo(), toEntityId); - Assert.assertEquals(relation.getTypeGroup().name(), relationUpdateMsg.getTypeGroup()); + Assert.assertEquals(deviceToAssetRelation.getTypeGroup().name(), relationUpdateMsg.getTypeGroup()); } } From 026f1748127db2da4cd944ad9c8b35d607b6b2b6 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Nov 2023 17:29:01 +0100 Subject: [PATCH 18/18] fixed rule-engine stats --- ...tomOAuth2AuthorizationRequestResolver.java | 2 + .../TbRuleEngineSecurityConfiguration.java | 44 +++++++++++++++++++ ...eOAuth2AuthorizationRequestRepository.java | 2 + 3 files changed, 48 insertions(+) create mode 100644 application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java diff --git a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java index 4d227345b1..e0d5d3e04e 100644 --- a/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java +++ b/application/src/main/java/org/thingsboard/server/config/CustomOAuth2AuthorizationRequestResolver.java @@ -38,6 +38,7 @@ import org.springframework.web.util.UriComponentsBuilder; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.dao.oauth2.OAuth2Service; +import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.oauth2.TbOAuth2ParameterNames; import org.thingsboard.server.service.security.model.token.OAuth2AppTokenFactory; import org.thingsboard.server.utils.MiscUtils; @@ -51,6 +52,7 @@ import java.util.HashMap; import java.util.Map; import java.util.UUID; +@TbCoreComponent @Service @Slf4j public class CustomOAuth2AuthorizationRequestResolver implements OAuth2AuthorizationRequestResolver { diff --git a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java new file mode 100644 index 0000000000..7b3a752f57 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java @@ -0,0 +1,44 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.security.SecurityProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.annotation.Order; +import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.web.SecurityFilterChain; + +@Configuration +@EnableWebSecurity +@EnableGlobalMethodSecurity(prePostEnabled = true) +@Order(SecurityProperties.BASIC_AUTH_ORDER) +@ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") +public class TbRuleEngineSecurityConfiguration { + + @Bean + SecurityFilterChain filterChain(HttpSecurity http) throws Exception { + http.headers().cacheControl().and().frameOptions().disable() + .and().cors().and().csrf().disable() + .authorizeRequests() + .antMatchers("/actuator/prometheus").permitAll() + .anyRequest().authenticated(); + return http.build(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java index d63f759dd3..98ccb2c1be 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/HttpCookieOAuth2AuthorizationRequestRepository.java @@ -18,11 +18,13 @@ package org.thingsboard.server.service.security.auth.oauth2; import org.springframework.security.oauth2.client.web.AuthorizationRequestRepository; import org.springframework.security.oauth2.core.endpoint.OAuth2AuthorizationRequest; import org.springframework.stereotype.Component; +import org.thingsboard.server.queue.util.TbCoreComponent; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @Component +@TbCoreComponent public class HttpCookieOAuth2AuthorizationRequestRepository implements AuthorizationRequestRepository { public static final String OAUTH2_AUTHORIZATION_REQUEST_COOKIE_NAME = "oauth2_auth_request"; public static final String PREV_URI_PARAMETER = "prevUri";