From 464265cf6e3902f85447044e6ee569ff460c0e11 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Jul 2020 13:12:35 +0300 Subject: [PATCH 01/39] fix bug with userLoginCaseSensitive after updating email --- .../java/org/thingsboard/server/dao/user/UserServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index 914ba5d3ec..f07e939cda 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -114,7 +114,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic public User saveUser(User user) { log.trace("Executing saveUser [{}]", user); userValidator.validate(user, User::getTenantId); - if (user.getId() == null && !userLoginCaseSensitive) { + if (!userLoginCaseSensitive) { user.setEmail(user.getEmail().toLowerCase()); } User savedUser = userDao.save(user.getTenantId(), user); From a400bdb905bdc5e01f6504a9f368e9e59264588b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 30 Jul 2020 18:46:37 +0300 Subject: [PATCH 02/39] added help link to azure iot hub node --- ui/src/app/help/help-links.constant.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ui/src/app/help/help-links.constant.js b/ui/src/app/help/help-links.constant.js index 8985b15f67..b1227fc9af 100644 --- a/ui/src/app/help/help-links.constant.js +++ b/ui/src/app/help/help-links.constant.js @@ -44,6 +44,7 @@ var ruleNodeClazzHelpLinkMap = { 'org.thingsboard.rule.engine.aws.sqs.TbSqsNode': 'ruleNodeAwsSqs', 'org.thingsboard.rule.engine.kafka.TbKafkaNode': 'ruleNodeKafka', 'org.thingsboard.rule.engine.mqtt.TbMqttNode': 'ruleNodeMqtt', + 'org.thingsboard.rule.engine.mqtt.azure.TbAzureIotHubNode': 'ruleNodeAzureIotHub', 'org.thingsboard.rule.engine.rabbitmq.TbRabbitMqNode': 'ruleNodeRabbitMq', 'org.thingsboard.rule.engine.rest.TbRestApiCallNode': 'ruleNodeRestApiCall', 'org.thingsboard.rule.engine.mail.TbSendEmailNode': 'ruleNodeSendEmail' @@ -88,6 +89,7 @@ export default angular.module('thingsboard.help', []) ruleNodeAwsSqs: helpBaseUrl + "/docs/user-guide/rule-engine-2-0/external-nodes/#aws-sqs-node", ruleNodeKafka: helpBaseUrl + "/docs/user-guide/rule-engine-2-0/external-nodes/#kafka-node", ruleNodeMqtt: helpBaseUrl + "/docs/user-guide/rule-engine-2-0/external-nodes/#mqtt-node", + ruleNodeAzureIotHub: helpBaseUrl + "/docs/user-guide/rule-engine-2-0/external-nodes/#azure-iot-hub-node", ruleNodeRabbitMq: helpBaseUrl + "/docs/user-guide/rule-engine-2-0/external-nodes/#rabbitmq-node", ruleNodeRestApiCall: helpBaseUrl + "/docs/user-guide/rule-engine-2-0/external-nodes/#rest-api-call-node", ruleNodeSendEmail: helpBaseUrl + "/docs/user-guide/rule-engine-2-0/external-nodes/#send-email-node", From ac007bea24ef4244190690a173c6ef323cb0258e Mon Sep 17 00:00:00 2001 From: Dima Landiak Date: Tue, 28 Jul 2020 19:18:03 +0300 Subject: [PATCH 03/39] device swap feature init --- .../server/controller/BaseController.java | 28 +++++++++- .../server/controller/DeviceController.java | 56 +++++++++++++++++++ .../server/dao/audit/AuditLogService.java | 3 +- .../server/dao/device/DeviceService.java | 4 +- .../server/dao/event/EventService.java | 2 + .../server/dao/relation/RelationService.java | 3 +- .../server/common/data/DataConstants.java | 2 + .../server/common/data/audit/ActionType.java | 4 +- .../server/dao/audit/AuditLogDao.java | 3 +- .../server/dao/audit/AuditLogServiceImpl.java | 39 ++++++++++--- .../dao/audit/DummyAuditLogServiceImpl.java | 5 +- .../server/dao/device/CassandraDeviceDao.java | 10 ++++ .../server/dao/device/DeviceDao.java | 17 ++++++ .../server/dao/device/DeviceServiceImpl.java | 49 +++++++++++++++- .../server/dao/event/BaseEventService.java | 19 +++++++ .../dao/relation/BaseRelationService.java | 16 ++++++ .../dao/sql/device/DeviceRepository.java | 3 + .../server/dao/sql/device/JpaDeviceDao.java | 10 ++++ .../engine/filter/TbMsgTypeSwitchNode.java | 13 ++++- ui/src/app/common/types.constant.js | 6 ++ ui/src/app/locale/locale.constant-en_US.json | 4 +- 21 files changed, 272 insertions(+), 24 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index f0e93f5662..95dcc7c64c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -26,7 +27,6 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.web.bind.annotation.ExceptionHandler; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.DashboardInfo; @@ -39,12 +39,12 @@ import org.thingsboard.server.common.data.HasTenantId; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; -import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.alarm.AlarmInfo; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; @@ -595,6 +595,12 @@ public abstract class BaseController { case ALARM_CLEAR: msgType = DataConstants.ALARM_CLEAR; break; + case SWAPPED_FROM_TENANT: + msgType = DataConstants.ENTITY_SWAPPED_FROM; + break; + case SWAPPED_TO_TENANT: + msgType = DataConstants.ENTITY_SWAPPED_TO; + break; } if (!StringUtils.isEmpty(msgType)) { try { @@ -614,6 +620,16 @@ public abstract class BaseController { String strCustomerName = extractParameter(String.class, 2, additionalInfo); metaData.putValue("unassignedCustomerId", strCustomerId); metaData.putValue("unassignedCustomerName", strCustomerName); + } else if (actionType == ActionType.SWAPPED_FROM_TENANT) { + String strTenantId = extractParameter(String.class, 0, additionalInfo); + String strTenantName = extractParameter(String.class, 1, additionalInfo); + metaData.putValue("swappedFromTenantId", strTenantId); + metaData.putValue("swappedFromTenantName", strTenantName); + } else if (actionType == ActionType.SWAPPED_TO_TENANT) { + String strTenantId = extractParameter(String.class, 0, additionalInfo); + String strTenantName = extractParameter(String.class, 1, additionalInfo); + metaData.putValue("swappedToTenantId", strTenantId); + metaData.putValue("swappedToTenantName", strTenantName); } ObjectNode entityNode; if (entity != null) { @@ -677,5 +693,13 @@ public abstract class BaseController { return result; } + protected String entityToStr(E entity) { + try { + return json.writeValueAsString(json.valueToTree(entity)); + } catch (JsonProcessingException e) { + log.warn("[{}] Failed to convert entity to string!", entity, e); + } + return null; + } } diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index f6f327d7d9..61505ddc90 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -39,8 +39,10 @@ import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.device.DeviceSearchQuery; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; @@ -48,6 +50,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.msg.TbMsg; +import org.thingsboard.server.common.msg.TbMsgDataType; +import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.exception.IncorrectParameterException; @@ -70,6 +75,7 @@ public class DeviceController extends BaseController { private static final String DEVICE_ID = "deviceId"; private static final String DEVICE_NAME = "deviceName"; + private static final String TENANT_ID = "tenantId"; @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") @RequestMapping(value = "/device/{deviceId}", method = RequestMethod.GET) @@ -481,4 +487,54 @@ public class DeviceController extends BaseController { } return DataConstants.DEFAULT_SECRET_KEY; } + + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + @RequestMapping(value = "/tenant/{tenantId}/device/{deviceId}", method = RequestMethod.POST) + @ResponseBody + public Device swapDevice(@PathVariable(TENANT_ID) String strTenantId, + @PathVariable(DEVICE_ID) String strDeviceId) throws ThingsboardException { + checkParameter(TENANT_ID, strTenantId); + checkParameter(DEVICE_ID, strDeviceId); + try { + DeviceId deviceId = new DeviceId(toUUID(strDeviceId)); + Device device = checkDeviceId(deviceId, Operation.WRITE); + + TenantId newTenantId = new TenantId(toUUID(strTenantId)); + Tenant newTenant = tenantService.findTenantById(newTenantId); + if (newTenant == null) { + throw new ThingsboardException("Could not find the specified Tenant!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); + } + + Device swappedDevice = deviceService.swapDevice(newTenantId, device); + + logEntityAction(getCurrentUser(), deviceId, swappedDevice, + swappedDevice.getCustomerId(), + ActionType.SWAPPED_TO_TENANT, null, strTenantId, newTenant.getName()); + + Tenant currentTenant = tenantService.findTenantById(getTenantId()); + pushSwappedFromNotification(currentTenant, newTenantId, swappedDevice); + + return swappedDevice; + } catch (Exception e) { + logEntityAction(getCurrentUser(), emptyId(EntityType.DEVICE), null, + null, + ActionType.SWAPPED_TO_TENANT, e, strTenantId); + throw handleException(e); + } + } + + private void pushSwappedFromNotification(Tenant currentTenant, TenantId newTenantId, Device swappedDevice) { + String data = entityToStr(swappedDevice); + if (data != null) { + TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_SWAPPED_FROM, swappedDevice.getId(), getMetaDataForSwappedFrom(currentTenant), TbMsgDataType.JSON, data); + tbClusterService.pushMsgToRuleEngine(newTenantId, swappedDevice.getId(), tbMsg, null); + } + } + + private TbMsgMetaData getMetaDataForSwappedFrom(Tenant tenant) { + TbMsgMetaData metaData = new TbMsgMetaData(); + metaData.putValue("swappedFromTenantId", tenant.getId().getId().toString()); + metaData.putValue("swappedFromTenantName", tenant.getName()); + return metaData; + } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java index 1103f25c2e..f54cda6681 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java @@ -16,14 +16,12 @@ package org.thingsboard.server.dao.audit; import com.google.common.util.concurrent.ListenableFuture; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.AuditLog; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; @@ -50,4 +48,5 @@ public interface AuditLogService { ActionType actionType, Exception e, Object... additionalInfo); + void removeAuditLogs(TenantId tenantId, EntityId entityId); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index a6e06616b2..20516be837 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.page.TextPageLink; import java.util.List; public interface DeviceService { - + Device findDeviceById(TenantId tenantId, DeviceId deviceId); ListenableFuture findDeviceByIdAsync(TenantId tenantId, DeviceId deviceId); @@ -65,4 +65,6 @@ public interface DeviceService { ListenableFuture> findDeviceTypesByTenantId(TenantId tenantId); + Device swapDevice(TenantId tenantId, Device device); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java index 1768e040d5..99f5e8d537 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/event/EventService.java @@ -41,4 +41,6 @@ public interface EventService { List findLatestEvents(TenantId tenantId, EntityId entityId, String eventType, int limit); + void removeEvents(TenantId tenantId, EntityId entityId); + } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java index a54eecf55e..2e909b3460 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/relation/RelationService.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.relation.EntityRelationsQuery; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import java.util.List; -import java.util.concurrent.ExecutionException; /** * Created by ashvayka on 27.04.17. @@ -77,6 +76,8 @@ public interface RelationService { ListenableFuture> findInfoByQuery(TenantId tenantId, EntityRelationsQuery query); + void removeRelations(TenantId tenantId, EntityId entityId); + // TODO: This method may be useful for some validations in the future // ListenableFuture checkRecursiveRelation(EntityId from, EntityId to); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index afb0dbeba6..22b49552c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -57,6 +57,8 @@ public class DataConstants { public static final String ATTRIBUTES_DELETED = "ATTRIBUTES_DELETED"; public static final String ALARM_ACK = "ALARM_ACK"; public static final String ALARM_CLEAR = "ALARM_CLEAR"; + public static final String ENTITY_SWAPPED_FROM = "ENTITY_SWAPPED_FROM"; + public static final String ENTITY_SWAPPED_TO = "ENTITY_SWAPPED_TO"; public static final String RPC_CALL_FROM_SERVER_TO_DEVICE = "RPC_CALL_FROM_SERVER_TO_DEVICE"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java index 03aeeba1d5..70909dc331 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java @@ -40,7 +40,9 @@ public enum ActionType { ALARM_CLEAR(false), LOGIN(false), LOGOUT(false), - LOCKOUT(false); + LOCKOUT(false), + SWAPPED_FROM_TENANT(false), + SWAPPED_TO_TENANT(false); private final boolean isRead; diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java index ba6b6841fb..f4efc2827f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogDao.java @@ -22,11 +22,12 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.dao.Dao; import java.util.List; import java.util.UUID; -public interface AuditLogDao { +public interface AuditLogDao extends Dao { ListenableFuture saveByTenantId(AuditLog auditLog); diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java index 902162ff84..72b8272c69 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java @@ -28,7 +28,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.audit.ActionStatus; @@ -38,7 +37,6 @@ import org.thingsboard.server.common.data.id.AuditLogId; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.page.TimePageData; @@ -53,6 +51,8 @@ import org.thingsboard.server.dao.service.DataValidator; import java.io.PrintWriter; import java.io.StringWriter; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -117,8 +117,8 @@ public class AuditLogServiceImpl implements AuditLogService { @Override public ListenableFuture> - logEntityAction(TenantId tenantId, CustomerId customerId, UserId userId, String userName, I entityId, E entity, - ActionType actionType, Exception e, Object... additionalInfo) { + logEntityAction(TenantId tenantId, CustomerId customerId, UserId userId, String userName, I entityId, E entity, + ActionType actionType, Exception e, Object... additionalInfo) { if (canLog(entityId.getEntityType(), actionType)) { JsonNode actionData = constructActionData(entityId, entity, actionType, additionalInfo); ActionStatus actionStatus = ActionStatus.SUCCESS; @@ -129,7 +129,8 @@ public class AuditLogServiceImpl implements AuditLogService { } else { try { entityName = entityService.fetchEntityNameAsync(tenantId, entityId).get(); - } catch (Exception ex) {} + } catch (Exception ex) { + } } if (e != null) { actionStatus = ActionStatus.FAILURE; @@ -157,16 +158,36 @@ public class AuditLogServiceImpl implements AuditLogService { } } + @Override + public void removeAuditLogs(TenantId tenantId, EntityId entityId) { + List auditLogs = new ArrayList<>(); + TimePageData auditLogPageData; + TimePageLink auditLogPageLink = new TimePageLink(1000); + do { + auditLogPageData = findAuditLogsByTenantIdAndEntityId(tenantId, entityId, + new ArrayList<>(Arrays.asList(ActionType.values())), auditLogPageLink); + auditLogs.addAll(auditLogPageData.getData()); + if (auditLogPageData.hasNext()) { + auditLogPageLink = auditLogPageData.getNextPageLink(); + } + } while (auditLogPageData.hasNext()); + + for (AuditLog auditLog : auditLogs) { + auditLogDao.removeById(tenantId, auditLog.getUuidId()); + } + } + private JsonNode constructActionData(I entityId, E entity, - ActionType actionType, - Object... additionalInfo) { + ActionType actionType, + Object... additionalInfo) { ObjectNode actionData = objectMapper.createObjectNode(); - switch(actionType) { + switch (actionType) { case ADDED: case UPDATED: case ALARM_ACK: case ALARM_CLEAR: case RELATIONS_DELETED: + case SWAPPED_TO_TENANT: if (entity != null) { ObjectNode entityNode = objectMapper.valueToTree(entity); if (entityId.getEntityType() == EntityType.DASHBOARD) { @@ -208,7 +229,7 @@ public class AuditLogServiceImpl implements AuditLogService { scope = extractParameter(String.class, 0, additionalInfo); actionData.put("scope", scope); List keys = extractParameter(List.class, 1, additionalInfo); - ArrayNode attrsArrayNode = actionData.putArray("attributes"); + ArrayNode attrsArrayNode = actionData.putArray("attributes"); if (keys != null) { keys.forEach(attrsArrayNode::add); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java index 93d0a5ca4e..c677b4368b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java @@ -18,14 +18,12 @@ package org.thingsboard.server.dao.audit; import com.google.common.util.concurrent.ListenableFuture; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.audit.AuditLog; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.id.UUIDBased; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.TimePageData; import org.thingsboard.server.common.data.page.TimePageLink; @@ -61,4 +59,7 @@ public class DummyAuditLogServiceImpl implements AuditLogService { return null; } + @Override + public void removeAuditLogs(TenantId tenantId, EntityId entityId) { + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java index a01725fe52..55f83a4aae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java @@ -189,4 +189,14 @@ public class CassandraDeviceDao extends CassandraAbstractSearchTextDao findDeviceByTenantIdAndIdAsync(TenantId tenantId, UUID id) { + return findByIdAsync(tenantId, id); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java index 930c5a5126..4d5cf09775 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceDao.java @@ -115,4 +115,21 @@ public interface DeviceDao extends Dao { * @return the list of tenant device type objects */ ListenableFuture> findTenantDeviceTypesAsync(UUID tenantId); + + /** + * Find devices by tenantId and device id. + * @param tenantId the tenant Id + * @param id the device Id + * @return the device object + */ + Device findDeviceByTenantIdAndId(TenantId tenantId, UUID id); + + /** + * Find devices by tenantId and device id. + * @param tenantId tenantId the tenantId + * @param id the deviceId + * @return the device object + */ + ListenableFuture findDeviceByTenantIdAndIdAsync(TenantId tenantId, UUID id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index db998b7328..6b27e29425 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -28,6 +28,8 @@ import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.CacheEvict; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; @@ -46,9 +48,11 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.entityview.EntityViewService; +import org.thingsboard.server.dao.event.EventService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; @@ -97,18 +101,32 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe @Autowired private CacheManager cacheManager; + @Autowired + private EventService eventService; + + @Autowired + private AuditLogService auditLogService; + @Override public Device findDeviceById(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceById [{}]", deviceId); validateId(deviceId, INCORRECT_DEVICE_ID + deviceId); - return deviceDao.findById(tenantId, deviceId.getId()); + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return deviceDao.findById(tenantId, deviceId.getId()); + } else { + return deviceDao.findDeviceByTenantIdAndId(tenantId, deviceId.getId()); + } } @Override public ListenableFuture findDeviceByIdAsync(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceById [{}]", deviceId); validateId(deviceId, INCORRECT_DEVICE_ID + deviceId); - return deviceDao.findByIdAsync(tenantId, deviceId.getId()); + if (TenantId.SYS_TENANT_ID.equals(tenantId)) { + return deviceDao.findByIdAsync(tenantId, deviceId.getId()); + } else { + return deviceDao.findDeviceByTenantIdAndIdAsync(tenantId, deviceId.getId()); + } } @Cacheable(cacheNames = DEVICE_CACHE, key = "{#tenantId, #name}") @@ -317,6 +335,33 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe }, MoreExecutors.directExecutor()); } + @Transactional + @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.name}") + @Override + public Device swapDevice(TenantId tenantId, Device device) { + log.trace("Executing swapDevice [{}]", device); + + try { + List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), device.getId()).get(); + if (!CollectionUtils.isEmpty(entityViews)) { + throw new DataValidationException("Can't swap device that is assigned to entity views!"); + } + } catch (ExecutionException | InterruptedException e) { + log.error("Exception while finding entity views for deviceId [{}]", device.getId(), e); + throw new RuntimeException("Exception while finding entity views for deviceId [" + device.getId() + "]", e); + } + + eventService.removeEvents(device.getTenantId(), device.getId()); + + relationService.removeRelations(device.getTenantId(), device.getId()); + + auditLogService.removeAuditLogs(device.getTenantId(), device.getId()); + + device.setTenantId(tenantId); + device.setCustomerId(null); + return doSaveDevice(device, null); + } + private DataValidator deviceValidator = new DataValidator() { diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 0f777d889f..50d49e75ad 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; +import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -94,6 +95,24 @@ public class BaseEventService implements EventService { return eventDao.findLatestEvents(tenantId.getId(), entityId, eventType, limit); } + @Override + public void removeEvents(TenantId tenantId, EntityId entityId) { + List events = new ArrayList<>(); + TimePageData eventPageData; + TimePageLink eventPageLink = new TimePageLink(1000); + do { + eventPageData = findEvents(tenantId, entityId, eventPageLink); + events.addAll(eventPageData.getData()); + if (eventPageData.hasNext()) { + eventPageLink = eventPageData.getNextPageLink(); + } + } while (eventPageData.hasNext()); + + for (Event event : events) { + eventDao.removeById(tenantId, event.getUuidId()); + } + } + private DataValidator eventValidator = new DataValidator() { @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java index 22b2543489..53ca684517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/BaseRelationService.java @@ -506,6 +506,22 @@ public class BaseRelationService implements RelationService { }, MoreExecutors.directExecutor()); } + @Override + public void removeRelations(TenantId tenantId, EntityId entityId) { + Cache cache = cacheManager.getCache(RELATIONS_CACHE); + + List relations = new ArrayList<>(); + for (RelationTypeGroup relationTypeGroup : RelationTypeGroup.values()) { + relations.addAll(findByFrom(tenantId, entityId, relationTypeGroup)); + relations.addAll(findByTo(tenantId, entityId, relationTypeGroup)); + } + + for (EntityRelation relation : relations) { + cacheEviction(relation, cache); + deleteRelation(tenantId, relation); + } + } + protected void validate(EntityRelation relation) { if (relation == null) { throw new DataValidationException("Relation type should be specified!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java index 16ff6f1b52..baf34dc59b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceRepository.java @@ -85,4 +85,7 @@ public interface DeviceRepository extends CrudRepository { List findDevicesByTenantIdAndCustomerIdAndIdIn(String tenantId, String customerId, List deviceIds); List findDevicesByTenantIdAndIdIn(String tenantId, List deviceIds); + + DeviceEntity findByTenantIdAndId(String tenantId, String id); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java index 34d59036a0..57d0ef6bb4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceDao.java @@ -138,6 +138,16 @@ public class JpaDeviceDao extends JpaAbstractSearchTextDao return service.submit(() -> convertTenantDeviceTypesToDto(tenantId, deviceRepository.findTenantDeviceTypes(fromTimeUUID(tenantId)))); } + @Override + public Device findDeviceByTenantIdAndId(TenantId tenantId, UUID id) { + return DaoUtil.getData(deviceRepository.findByTenantIdAndId(fromTimeUUID(tenantId.getId()), fromTimeUUID(id))); + } + + @Override + public ListenableFuture findDeviceByTenantIdAndIdAsync(TenantId tenantId, UUID id) { + return service.submit(() -> DaoUtil.getData(deviceRepository.findByTenantIdAndId(fromTimeUUID(tenantId.getId()), fromTimeUUID(id)))); + } + private List convertTenantDeviceTypesToDto(UUID tenantId, List types) { List list = Collections.emptyList(); if (types != null && !types.isEmpty()) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index 07a0508a87..62e75cd72a 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -16,8 +16,13 @@ package org.thingsboard.rule.engine.filter; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.rule.engine.api.EmptyNodeConfiguration; +import org.thingsboard.rule.engine.api.RuleNode; +import org.thingsboard.rule.engine.api.TbContext; +import org.thingsboard.rule.engine.api.TbNode; +import org.thingsboard.rule.engine.api.TbNodeConfiguration; +import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.util.TbNodeUtils; -import org.thingsboard.rule.engine.api.*; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.msg.TbMsg; @@ -30,7 +35,7 @@ import org.thingsboard.server.common.msg.session.SessionMsgType; configClazz = EmptyNodeConfiguration.class, relationTypes = {"Post attributes", "Post telemetry", "RPC Request from Device", "RPC Request to Device", "Activity Event", "Inactivity Event", "Connect Event", "Disconnect Event", "Entity Created", "Entity Updated", "Entity Deleted", "Entity Assigned", - "Entity Unassigned", "Attributes Updated", "Attributes Deleted", "Alarm Acknowledged", "Alarm Cleared", "Other"}, + "Entity Unassigned", "Attributes Updated", "Attributes Deleted", "Alarm Acknowledged", "Alarm Cleared", "Other", "Entity Swapped From", "Entity Swapped To"}, nodeDescription = "Route incoming messages by Message Type", nodeDetails = "Sends messages with message types \"Post attributes\", \"Post telemetry\", \"RPC Request\" etc. via corresponding chain, otherwise Other chain is used.", uiResources = {"static/rulenode/rulenode-core-config.js"}, @@ -81,6 +86,10 @@ public class TbMsgTypeSwitchNode implements TbNode { relationType = "Alarm Cleared"; } else if (msg.getType().equals(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE)) { relationType = "RPC Request to Device"; + } else if (msg.getType().equals(DataConstants.ENTITY_SWAPPED_FROM)) { + relationType = "Entity Swapped From"; + } else if (msg.getType().equals(DataConstants.ENTITY_SWAPPED_TO)) { + relationType = "Entity Swapped To"; } else { relationType = "Other"; } diff --git a/ui/src/app/common/types.constant.js b/ui/src/app/common/types.constant.js index 8285430c56..254313b7a0 100644 --- a/ui/src/app/common/types.constant.js +++ b/ui/src/app/common/types.constant.js @@ -222,6 +222,12 @@ export default angular.module('thingsboard.types', []) }, "LOCKOUT": { name: "audit-log.type-lockout" + }, + "SWAPPED_FROM_TENANT": { + name: "audit-log.type-swapped-from-tenant" + }, + "SWAPPED_TO_TENANT": { + name: "audit-log.type-swapped-to-tenant" } }, auditLogActionStatus: { diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index f6172d585f..7e81fd1a57 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -356,7 +356,9 @@ "action-data": "Action data", "failure-details": "Failure details", "search": "Search audit logs", - "clear-search": "Clear search" + "clear-search": "Clear search", + "type-swapped-from-tenant": "Swapped from Tenant", + "type-swapped-to-tenant": "Swapped to Tenant" }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?", From a167b39b446bf4ba5425f9f6d679f06fe2be7524 Mon Sep 17 00:00:00 2001 From: Dima Landiak Date: Thu, 30 Jul 2020 18:49:41 +0300 Subject: [PATCH 04/39] fixes --- .../controller/AbstractControllerTest.java | 23 +- .../controller/BaseDeviceControllerTest.java | 436 ++++++++++-------- .../server/dao/device/CassandraDeviceDao.java | 10 +- .../server/dao/device/DeviceServiceImpl.java | 5 +- 4 files changed, 282 insertions(+), 192 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index 5462bb12bd..ca1991ed12 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -79,8 +79,14 @@ import java.util.Comparator; import java.util.List; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; -import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; +import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.request; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.springframework.test.web.servlet.setup.MockMvcBuilders.webAppContextSetup; @ActiveProfiles("test") @@ -221,6 +227,7 @@ public abstract class AbstractControllerTest { } private Tenant savedDifferentTenant; + protected void loginDifferentTenant() throws Exception { loginSysAdmin(); Tenant tenant = new Tenant(); @@ -316,6 +323,10 @@ public abstract class AbstractControllerTest { return readResponse(doGet(urlTemplate, urlVariables).andExpect(status().isOk()), responseClass); } + protected T doGet(String urlTemplate, Class responseClass, ResultMatcher resultMatcher, Object... urlVariables) throws Exception { + return readResponse(doGet(urlTemplate, urlVariables).andExpect(resultMatcher), responseClass); + } + protected T doGetAsync(String urlTemplate, Class responseClass, Object... urlVariables) throws Exception { return readResponse(doGetAsync(urlTemplate, urlVariables).andExpect(status().isOk()), responseClass); } @@ -357,9 +368,9 @@ public abstract class AbstractControllerTest { return readResponse(doGet(urlTemplate, vars).andExpect(status().isOk()), responseType); } - protected T doGetTypedWithTimePageLink(String urlTemplate, TypeReference responseType, - TimePageLink pageLink, - Object... urlVariables) throws Exception { + protected T doGetTypedWithTimePageLink(String urlTemplate, TypeReference responseType, + TimePageLink pageLink, + Object... urlVariables) throws Exception { List pageLinkVariables = new ArrayList<>(); urlTemplate += "limit={limit}"; pageLinkVariables.add(pageLink.getLimit()); @@ -425,7 +436,7 @@ public abstract class AbstractControllerTest { return mockMvc.perform(postRequest); } - protected ResultActions doPostAsync(String urlTemplate, T content, Long timeout, String... params) throws Exception { + protected ResultActions doPostAsync(String urlTemplate, T content, Long timeout, String... params) throws Exception { MockHttpServletRequestBuilder postRequest = post(urlTemplate); setJwtToken(postRequest); String json = json(content); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index aa11c4ab40..5f4f8a3a03 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -15,74 +15,79 @@ */ package org.thingsboard.server.controller; -import static org.hamcrest.Matchers.containsString; -import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - import com.datastax.driver.core.utils.UUIDs; +import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.lang3.RandomStringUtils; -import org.thingsboard.server.common.data.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceCredentialsId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.page.TextPageData; import org.thingsboard.server.common.data.page.TextPageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.dao.model.ModelConstants; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import com.fasterxml.jackson.core.type.TypeReference; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsString; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; public abstract class BaseDeviceControllerTest extends AbstractControllerTest { - + private IdComparator idComparator = new IdComparator<>(); - + private Tenant savedTenant; private User tenantAdmin; - + @Before public void beforeTest() throws Exception { loginSysAdmin(); - + Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); savedTenant = doPost("/api/tenant", tenant, Tenant.class); Assert.assertNotNull(savedTenant); - + tenantAdmin = new User(); tenantAdmin.setAuthority(Authority.TENANT_ADMIN); tenantAdmin.setTenantId(savedTenant.getId()); tenantAdmin.setEmail("tenant2@thingsboard.org"); tenantAdmin.setFirstName("Joe"); tenantAdmin.setLastName("Downs"); - + tenantAdmin = createUserAndLogin(tenantAdmin, "testPassword1"); } - + @After public void afterTest() throws Exception { loginSysAdmin(); - - doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) - .andExpect(status().isOk()); + + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) + .andExpect(status().isOk()); } - + @Test public void testSaveDevice() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - + Assert.assertNotNull(savedDevice); Assert.assertNotNull(savedDevice.getId()); Assert.assertTrue(savedDevice.getCreatedTime() > 0); @@ -90,9 +95,9 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertNotNull(savedDevice.getCustomerId()); Assert.assertEquals(NULL_UUID, savedDevice.getCustomerId().getId()); Assert.assertEquals(device.getName(), savedDevice.getName()); - - DeviceCredentials deviceCredentials = - doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); + + DeviceCredentials deviceCredentials = + doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); Assert.assertNotNull(deviceCredentials); Assert.assertNotNull(deviceCredentials.getId()); @@ -100,10 +105,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(DeviceCredentialsType.ACCESS_TOKEN, deviceCredentials.getCredentialsType()); Assert.assertNotNull(deviceCredentials.getCredentialsId()); Assert.assertEquals(20, deviceCredentials.getCredentialsId().length()); - + savedDevice.setName("My new device"); doPost("/api/device", savedDevice, Device.class); - + Device foundDevice = doGet("/api/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(foundDevice.getName(), savedDevice.getName()); } @@ -115,10 +120,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); loginDifferentTenant(); - doPost("/api/device", savedDevice, Device.class, status().isForbidden()); + doPost("/api/device", savedDevice, Device.class, status().isNotFound()); deleteDifferentTenant(); } - + @Test public void testFindDeviceById() throws Exception { Device device = new Device(); @@ -133,26 +138,27 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { @Test public void testFindDeviceTypesByTenantId() throws Exception { List devices = new ArrayList<>(); - for (int i=0;i<3;i++) { + for (int i = 0; i < 3; i++) { Device device = new Device(); - device.setName("My device B"+i); + device.setName("My device B" + i); device.setType("typeB"); devices.add(doPost("/api/device", device, Device.class)); } - for (int i=0;i<7;i++) { + for (int i = 0; i < 7; i++) { Device device = new Device(); - device.setName("My device C"+i); + device.setName("My device C" + i); device.setType("typeC"); devices.add(doPost("/api/device", device, Device.class)); } - for (int i=0;i<9;i++) { + for (int i = 0; i < 9; i++) { Device device = new Device(); - device.setName("My device A"+i); + device.setName("My device A" + i); device.setType("typeA"); devices.add(doPost("/api/device", device, Device.class)); } List deviceTypes = doGetTyped("/api/device/types", - new TypeReference>(){}); + new TypeReference>() { + }); Assert.assertNotNull(deviceTypes); Assert.assertEquals(3, deviceTypes.size()); @@ -160,19 +166,19 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals("typeB", deviceTypes.get(1).getType()); Assert.assertEquals("typeC", deviceTypes.get(2).getType()); } - + @Test public void testDeleteDevice() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - - doDelete("/api/device/"+savedDevice.getId().getId().toString()) - .andExpect(status().isOk()); - doGet("/api/device/"+savedDevice.getId().getId().toString()) - .andExpect(status().isNotFound()); + doDelete("/api/device/" + savedDevice.getId().getId().toString()) + .andExpect(status().isOk()); + + doGet("/api/device/" + savedDevice.getId().getId().toString()) + .andExpect(status().isNotFound()); } @Test @@ -189,52 +195,52 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Device device = new Device(); device.setType("default"); doPost("/api/device", device) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Device name should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device name should be specified"))); } - + @Test public void testAssignUnassignDeviceToCustomer() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - + Customer customer = new Customer(); customer.setTitle("My customer"); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); - - Device assignedDevice = doPost("/api/customer/" + savedCustomer.getId().getId().toString() + + Device assignedDevice = doPost("/api/customer/" + savedCustomer.getId().getId().toString() + "/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(savedCustomer.getId(), assignedDevice.getCustomerId()); - + Device foundDevice = doGet("/api/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(savedCustomer.getId(), foundDevice.getCustomerId()); - Device unassignedDevice = + Device unassignedDevice = doDelete("/api/customer/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(ModelConstants.NULL_UUID, unassignedDevice.getCustomerId().getId()); - + foundDevice = doGet("/api/device/" + savedDevice.getId().getId().toString(), Device.class); Assert.assertEquals(ModelConstants.NULL_UUID, foundDevice.getCustomerId().getId()); } - + @Test public void testAssignDeviceToNonExistentCustomer() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - + doPost("/api/customer/" + UUIDs.timeBased().toString() + "/device/" + savedDevice.getId().getId().toString()) - .andExpect(status().isNotFound()); + .andExpect(status().isNotFound()); } - + @Test public void testAssignDeviceToCustomerFromDifferentTenant() throws Exception { loginSysAdmin(); - + Tenant tenant2 = new Tenant(); tenant2.setTitle("Different tenant"); Tenant savedTenant2 = doPost("/api/tenant", tenant2, Tenant.class); @@ -246,103 +252,103 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { tenantAdmin2.setEmail("tenant3@thingsboard.org"); tenantAdmin2.setFirstName("Joe"); tenantAdmin2.setLastName("Downs"); - + tenantAdmin2 = createUserAndLogin(tenantAdmin2, "testPassword1"); - + Customer customer = new Customer(); customer.setTitle("Different customer"); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); login(tenantAdmin.getEmail(), "testPassword1"); - + Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - + doPost("/api/customer/" + savedCustomer.getId().getId().toString() + "/device/" + savedDevice.getId().getId().toString()) - .andExpect(status().isForbidden()); - + .andExpect(status().isForbidden()); + loginSysAdmin(); - - doDelete("/api/tenant/"+savedTenant2.getId().getId().toString()) - .andExpect(status().isOk()); + + doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) + .andExpect(status().isOk()); } - + @Test public void testFindDeviceCredentialsByDeviceId() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = - doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); + DeviceCredentials deviceCredentials = + doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); Assert.assertEquals(savedDevice.getId(), deviceCredentials.getDeviceId()); } - + @Test public void testSaveDeviceCredentials() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = - doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); + DeviceCredentials deviceCredentials = + doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); Assert.assertEquals(savedDevice.getId(), deviceCredentials.getDeviceId()); deviceCredentials.setCredentialsType(DeviceCredentialsType.ACCESS_TOKEN); deviceCredentials.setCredentialsId("access_token"); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isOk()); - - DeviceCredentials foundDeviceCredentials = + .andExpect(status().isOk()); + + DeviceCredentials foundDeviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); - + Assert.assertEquals(deviceCredentials, foundDeviceCredentials); } - + @Test public void testSaveDeviceCredentialsWithEmptyDevice() throws Exception { DeviceCredentials deviceCredentials = new DeviceCredentials(); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isBadRequest()); + .andExpect(status().isBadRequest()); } - + @Test public void testSaveDeviceCredentialsWithEmptyCredentialsType() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); deviceCredentials.setCredentialsType(null); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Device credentials type should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device credentials type should be specified"))); } - + @Test public void testSaveDeviceCredentialsWithEmptyCredentialsId() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); deviceCredentials.setCredentialsId(null); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Device credentials id should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device credentials id should be specified"))); } - + @Test public void testSaveNonExistentDeviceCredentials() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); DeviceCredentials newDeviceCredentials = new DeviceCredentials(new DeviceCredentialsId(UUIDs.timeBased())); newDeviceCredentials.setCreatedTime(deviceCredentials.getCreatedTime()); @@ -350,29 +356,29 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { newDeviceCredentials.setCredentialsType(deviceCredentials.getCredentialsType()); newDeviceCredentials.setCredentialsId(deviceCredentials.getCredentialsId()); doPost("/api/device/credentials", newDeviceCredentials) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Unable to update non-existent device credentials"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Unable to update non-existent device credentials"))); } - + @Test public void testSaveDeviceCredentialsWithNonExistentDevice() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); - DeviceCredentials deviceCredentials = + DeviceCredentials deviceCredentials = doGet("/api/device/" + savedDevice.getId().getId().toString() + "/credentials", DeviceCredentials.class); deviceCredentials.setDeviceId(new DeviceId(UUIDs.timeBased())); doPost("/api/device/credentials", deviceCredentials) - .andExpect(status().isNotFound()); + .andExpect(status().isNotFound()); } @Test public void testFindTenantDevices() throws Exception { List devices = new ArrayList<>(); - for (int i=0;i<178;i++) { + for (int i = 0; i < 178; i++) { Device device = new Device(); - device.setName("Device"+i); + device.setName("Device" + i); device.setType("default"); devices.add(doPost("/api/device", device, Device.class)); } @@ -380,28 +386,29 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { TextPageLink pageLink = new TextPageLink(23); TextPageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/tenant/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/tenant/devices?", + new TypeReference>() { + }, pageLink); loadedDevices.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devices, idComparator); Collections.sort(loadedDevices, idComparator); - + Assert.assertEquals(devices, loadedDevices); } - + @Test public void testFindTenantDevicesByName() throws Exception { String title1 = "Device title 1"; List devicesTitle1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); @@ -409,38 +416,40 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { } String title2 = "Device title 2"; List devicesTitle2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); devicesTitle2.add(doPost("/api/device", device, Device.class)); } - + List loadedDevicesTitle1 = new ArrayList<>(); TextPageLink pageLink = new TextPageLink(15, title1); TextPageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/tenant/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/tenant/devices?", + new TypeReference>() { + }, pageLink); loadedDevicesTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devicesTitle1, idComparator); Collections.sort(loadedDevicesTitle1, idComparator); - + Assert.assertEquals(devicesTitle1, loadedDevicesTitle1); - + List loadedDevicesTitle2 = new ArrayList<>(); pageLink = new TextPageLink(4, title2); do { - pageData = doGetTypedWithPageLink("/api/tenant/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/tenant/devices?", + new TypeReference>() { + }, pageLink); loadedDevicesTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -449,28 +458,30 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Collections.sort(devicesTitle2, idComparator); Collections.sort(loadedDevicesTitle2, idComparator); - + Assert.assertEquals(devicesTitle2, loadedDevicesTitle2); - + for (Device device : loadedDevicesTitle1) { - doDelete("/api/device/"+device.getId().getId().toString()) - .andExpect(status().isOk()); + doDelete("/api/device/" + device.getId().getId().toString()) + .andExpect(status().isOk()); } - + pageLink = new TextPageLink(4, title1); - pageData = doGetTypedWithPageLink("/api/tenant/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/tenant/devices?", + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); - + for (Device device : loadedDevicesTitle2) { - doDelete("/api/device/"+device.getId().getId().toString()) - .andExpect(status().isOk()); + doDelete("/api/device/" + device.getId().getId().toString()) + .andExpect(status().isOk()); } - + pageLink = new TextPageLink(4, title2); - pageData = doGetTypedWithPageLink("/api/tenant/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/tenant/devices?", + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -480,10 +491,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title1 = "Device title 1"; String type1 = "typeA"; List devicesType1 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type1); @@ -492,10 +503,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title2 = "Device title 2"; String type2 = "typeB"; List devicesType2 = new ArrayList<>(); - for (int i=0;i<75;i++) { + for (int i = 0; i < 75; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type2); @@ -507,7 +518,8 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/tenant/devices?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedDevicesType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -523,7 +535,8 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/tenant/devices?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedDevicesType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -536,63 +549,66 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertEquals(devicesType2, loadedDevicesType2); for (Device device : loadedDevicesType1) { - doDelete("/api/device/"+device.getId().getId().toString()) + doDelete("/api/device/" + device.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/devices?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); for (Device device : loadedDevicesType2) { - doDelete("/api/device/"+device.getId().getId().toString()) + doDelete("/api/device/" + device.getId().getId().toString()) .andExpect(status().isOk()); } pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/tenant/devices?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } - + @Test public void testFindCustomerDevices() throws Exception { Customer customer = new Customer(); customer.setTitle("Test customer"); customer = doPost("/api/customer", customer, Customer.class); CustomerId customerId = customer.getId(); - + List devices = new ArrayList<>(); - for (int i=0;i<128;i++) { + for (int i = 0; i < 128; i++) { Device device = new Device(); - device.setName("Device"+i); + device.setName("Device" + i); device.setType("default"); device = doPost("/api/device", device, Device.class); - devices.add(doPost("/api/customer/" + customerId.getId().toString() - + "/device/" + device.getId().getId().toString(), Device.class)); + devices.add(doPost("/api/customer/" + customerId.getId().toString() + + "/device/" + device.getId().getId().toString(), Device.class)); } - + List loadedDevices = new ArrayList<>(); TextPageLink pageLink = new TextPageLink(23); TextPageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + new TypeReference>() { + }, pageLink); loadedDevices.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devices, idComparator); Collections.sort(loadedDevices, idComparator); - + Assert.assertEquals(devices, loadedDevices); } - + @Test public void testFindCustomerDevicesByName() throws Exception { Customer customer = new Customer(); @@ -602,53 +618,55 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title1 = "Device title 1"; List devicesTitle1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); device = doPost("/api/device", device, Device.class); - devicesTitle1.add(doPost("/api/customer/" + customerId.getId().toString() + devicesTitle1.add(doPost("/api/customer/" + customerId.getId().toString() + "/device/" + device.getId().getId().toString(), Device.class)); } String title2 = "Device title 2"; List devicesTitle2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType("default"); device = doPost("/api/device", device, Device.class); - devicesTitle2.add(doPost("/api/customer/" + customerId.getId().toString() + devicesTitle2.add(doPost("/api/customer/" + customerId.getId().toString() + "/device/" + device.getId().getId().toString(), Device.class)); } - + List loadedDevicesTitle1 = new ArrayList<>(); TextPageLink pageLink = new TextPageLink(15, title1); TextPageData pageData = null; do { - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + new TypeReference>() { + }, pageLink); loadedDevicesTitle1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); } } while (pageData.hasNext()); - + Collections.sort(devicesTitle1, idComparator); Collections.sort(loadedDevicesTitle1, idComparator); - + Assert.assertEquals(devicesTitle1, loadedDevicesTitle1); - + List loadedDevicesTitle2 = new ArrayList<>(); pageLink = new TextPageLink(4, title2); do { - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + new TypeReference>() { + }, pageLink); loadedDevicesTitle2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -657,28 +675,30 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Collections.sort(devicesTitle2, idComparator); Collections.sort(loadedDevicesTitle2, idComparator); - + Assert.assertEquals(devicesTitle2, loadedDevicesTitle2); - + for (Device device : loadedDevicesTitle1) { doDelete("/api/customer/device/" + device.getId().getId().toString()) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } - + pageLink = new TextPageLink(4, title1); - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); - + for (Device device : loadedDevicesTitle2) { doDelete("/api/customer/device/" + device.getId().getId().toString()) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } - + pageLink = new TextPageLink(4, title2); - pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", - new TypeReference>(){}, pageLink); + pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?", + new TypeReference>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } @@ -693,10 +713,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title1 = "Device title 1"; String type1 = "typeC"; List devicesType1 = new ArrayList<>(); - for (int i=0;i<125;i++) { + for (int i = 0; i < 125; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title1+suffix; + String name = title1 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type1); @@ -707,10 +727,10 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String title2 = "Device title 2"; String type2 = "typeD"; List devicesType2 = new ArrayList<>(); - for (int i=0;i<143;i++) { + for (int i = 0; i < 143; i++) { Device device = new Device(); String suffix = RandomStringUtils.randomAlphanumeric(15); - String name = title2+suffix; + String name = title2 + suffix; name = i % 2 == 0 ? name.toLowerCase() : name.toUpperCase(); device.setName(name); device.setType(type2); @@ -724,7 +744,8 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { TextPageData pageData = null; do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); loadedDevicesType1.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -740,7 +761,8 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); do { pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); loadedDevicesType2.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageData.getNextPageLink(); @@ -759,7 +781,8 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?type={type}&", - new TypeReference>(){}, pageLink, type1); + new TypeReference>() { + }, pageLink, type1); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); @@ -770,9 +793,56 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { pageLink = new TextPageLink(4); pageData = doGetTypedWithPageLink("/api/customer/" + customerId.getId().toString() + "/devices?type={type}&", - new TypeReference>(){}, pageLink, type2); + new TypeReference>() { + }, pageLink, type2); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(0, pageData.getData().size()); } + @Test + public void testSwapDeviceFromOneTenantToAnother() throws Exception { + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + Device savedDevice = doPost("/api/device", device, Device.class); + + Device anotherDevice = new Device(); + anotherDevice.setName("My device1"); + anotherDevice.setType("default"); + Device savedAnotherDevice = doPost("/api/device", anotherDevice, Device.class); + + EntityRelation relation = new EntityRelation(); + relation.setFrom(savedDevice.getId()); + relation.setTo(savedAnotherDevice.getId()); + relation.setTypeGroup(RelationTypeGroup.COMMON); + relation.setType("Contains"); + doPost("/api/relation", relation); + + loginSysAdmin(); + Tenant tenant = new Tenant(); + tenant.setTitle("Different tenant"); + Tenant savedDifferentTenant = doPost("/api/tenant", tenant, Tenant.class); + Assert.assertNotNull(savedDifferentTenant); + + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setTenantId(savedDifferentTenant.getId()); + user.setEmail("tenant9@thingsboard.org"); + user.setFirstName("Sam"); + user.setLastName("Downs"); + + createUserAndLogin(user, "testPassword1"); + + login("tenant2@thingsboard.org", "testPassword1"); + Device swappedDevice = doPost("/api/tenant/" + savedDifferentTenant.getId().getId() + "/device/" + savedDevice.getId().getId(), Device.class); + + doGet("/api/device/" + swappedDevice.getId().getId().toString(), Device.class, status().isNotFound()); + + login("tenant9@thingsboard.org", "testPassword1"); + + Device foundDevice1 = doGet("/api/device/" + swappedDevice.getId().getId().toString(), Device.class); + Assert.assertNotNull(foundDevice1); + + doGet("/api/relation?fromId=" + savedDevice.getId().getId() + "&fromType=DEVICE&relationType=Contains&toId=" + savedAnotherDevice.getId().getId() + "&toType=DEVICE", EntityRelation.class, status().isNotFound()); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java b/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java index 55f83a4aae..b010bbfdc2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/CassandraDeviceDao.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.EntitySubtypeEntity; +import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.nosql.DeviceEntity; import org.thingsboard.server.dao.nosql.CassandraAbstractSearchTextDao; import org.thingsboard.server.dao.util.NoSqlDao; @@ -191,12 +192,17 @@ public class CassandraDeviceDao extends CassandraAbstractSearchTextDao findDeviceByTenantIdAndIdAsync(TenantId tenantId, UUID id) { - return findByIdAsync(tenantId, id); + Select.Where query = select().from(getColumnFamilyName()).where(eq(ModelConstants.TENANT_ID_PROPERTY, tenantId.getId())).and(eq(ModelConstants.ID_PROPERTY, id)); + log.trace("Execute query {}", query); + return findOneByStatementAsync(tenantId, query); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 6b27e29425..7abcaa73c3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -355,7 +355,10 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe relationService.removeRelations(device.getTenantId(), device.getId()); - auditLogService.removeAuditLogs(device.getTenantId(), device.getId()); + // TODO: 30/07/2020 implement for Cassandra + if (sqlDatabaseUsed) { + auditLogService.removeAuditLogs(device.getTenantId(), device.getId()); + } device.setTenantId(tenantId); device.setCustomerId(null); From 287e4bbb5ecb1e42724a976124f787311f31151b Mon Sep 17 00:00:00 2001 From: Dima Landiak Date: Thu, 30 Jul 2020 19:21:07 +0300 Subject: [PATCH 05/39] fix test --- .../server/controller/BaseDeviceControllerTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index 5f4f8a3a03..48cf397168 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -844,5 +844,9 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Assert.assertNotNull(foundDevice1); doGet("/api/relation?fromId=" + savedDevice.getId().getId() + "&fromType=DEVICE&relationType=Contains&toId=" + savedAnotherDevice.getId().getId() + "&toType=DEVICE", EntityRelation.class, status().isNotFound()); + + loginSysAdmin(); + doDelete("/api/tenant/" + savedDifferentTenant.getId().getId().toString()) + .andExpect(status().isOk()); } } From fea2f8799ed2a86b062e038c3013aa1ebc3b57b3 Mon Sep 17 00:00:00 2001 From: Dima Landiak Date: Fri, 31 Jul 2020 14:43:25 +0300 Subject: [PATCH 06/39] improvements --- .../server/controller/BaseController.java | 20 +++++------ .../server/controller/DeviceController.java | 34 +++++++++---------- .../security/permission/Operation.java | 2 +- .../controller/BaseDeviceControllerTest.java | 10 +++--- .../server/dao/audit/AuditLogService.java | 2 -- .../server/dao/device/DeviceService.java | 2 +- .../server/common/data/DataConstants.java | 4 +-- .../server/common/data/audit/ActionType.java | 4 +-- .../server/dao/asset/BaseAssetService.java | 2 +- .../server/dao/audit/AuditLogServiceImpl.java | 23 +------------ .../dao/audit/DummyAuditLogServiceImpl.java | 4 --- .../server/dao/device/DeviceServiceImpl.java | 17 +++------- .../server/dao/event/BaseEventService.java | 10 ++---- .../engine/filter/TbMsgTypeSwitchNode.java | 10 +++--- ui/src/app/common/types.constant.js | 8 ++--- ui/src/app/locale/locale.constant-en_US.json | 4 +-- 16 files changed, 58 insertions(+), 98 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/BaseController.java b/application/src/main/java/org/thingsboard/server/controller/BaseController.java index 95dcc7c64c..48e528b0d8 100644 --- a/application/src/main/java/org/thingsboard/server/controller/BaseController.java +++ b/application/src/main/java/org/thingsboard/server/controller/BaseController.java @@ -595,11 +595,11 @@ public abstract class BaseController { case ALARM_CLEAR: msgType = DataConstants.ALARM_CLEAR; break; - case SWAPPED_FROM_TENANT: - msgType = DataConstants.ENTITY_SWAPPED_FROM; + case ASSIGNED_FROM_TENANT: + msgType = DataConstants.ENTITY_ASSIGNED_FROM_TENANT; break; - case SWAPPED_TO_TENANT: - msgType = DataConstants.ENTITY_SWAPPED_TO; + case ASSIGNED_TO_TENANT: + msgType = DataConstants.ENTITY_ASSIGNED_TO_TENANT; break; } if (!StringUtils.isEmpty(msgType)) { @@ -620,16 +620,16 @@ public abstract class BaseController { String strCustomerName = extractParameter(String.class, 2, additionalInfo); metaData.putValue("unassignedCustomerId", strCustomerId); metaData.putValue("unassignedCustomerName", strCustomerName); - } else if (actionType == ActionType.SWAPPED_FROM_TENANT) { + } else if (actionType == ActionType.ASSIGNED_FROM_TENANT) { String strTenantId = extractParameter(String.class, 0, additionalInfo); String strTenantName = extractParameter(String.class, 1, additionalInfo); - metaData.putValue("swappedFromTenantId", strTenantId); - metaData.putValue("swappedFromTenantName", strTenantName); - } else if (actionType == ActionType.SWAPPED_TO_TENANT) { + metaData.putValue("assignedFromTenantId", strTenantId); + metaData.putValue("assignedFromTenantName", strTenantName); + } else if (actionType == ActionType.ASSIGNED_TO_TENANT) { String strTenantId = extractParameter(String.class, 0, additionalInfo); String strTenantName = extractParameter(String.class, 1, additionalInfo); - metaData.putValue("swappedToTenantId", strTenantId); - metaData.putValue("swappedToTenantName", strTenantName); + metaData.putValue("assignedToTenantId", strTenantId); + metaData.putValue("assignedToTenantName", strTenantName); } ObjectNode entityNode; if (entity != null) { diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 61505ddc90..4f226d2d4c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -491,13 +491,13 @@ public class DeviceController extends BaseController { @PreAuthorize("hasAuthority('TENANT_ADMIN')") @RequestMapping(value = "/tenant/{tenantId}/device/{deviceId}", method = RequestMethod.POST) @ResponseBody - public Device swapDevice(@PathVariable(TENANT_ID) String strTenantId, - @PathVariable(DEVICE_ID) String strDeviceId) throws ThingsboardException { + public Device assignDeviceToTenant(@PathVariable(TENANT_ID) String strTenantId, + @PathVariable(DEVICE_ID) String strDeviceId) throws ThingsboardException { checkParameter(TENANT_ID, strTenantId); checkParameter(DEVICE_ID, strDeviceId); try { DeviceId deviceId = new DeviceId(toUUID(strDeviceId)); - Device device = checkDeviceId(deviceId, Operation.WRITE); + Device device = checkDeviceId(deviceId, Operation.ASSIGN_TO_TENANT); TenantId newTenantId = new TenantId(toUUID(strTenantId)); Tenant newTenant = tenantService.findTenantById(newTenantId); @@ -505,36 +505,36 @@ public class DeviceController extends BaseController { throw new ThingsboardException("Could not find the specified Tenant!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } - Device swappedDevice = deviceService.swapDevice(newTenantId, device); + Device assignedDevice = deviceService.assignDeviceToTenant(newTenantId, device); - logEntityAction(getCurrentUser(), deviceId, swappedDevice, - swappedDevice.getCustomerId(), - ActionType.SWAPPED_TO_TENANT, null, strTenantId, newTenant.getName()); + logEntityAction(getCurrentUser(), deviceId, assignedDevice, + assignedDevice.getCustomerId(), + ActionType.ASSIGNED_TO_TENANT, null, strTenantId, newTenant.getName()); Tenant currentTenant = tenantService.findTenantById(getTenantId()); - pushSwappedFromNotification(currentTenant, newTenantId, swappedDevice); + pushAssignedFromNotification(currentTenant, newTenantId, assignedDevice); - return swappedDevice; + return assignedDevice; } catch (Exception e) { logEntityAction(getCurrentUser(), emptyId(EntityType.DEVICE), null, null, - ActionType.SWAPPED_TO_TENANT, e, strTenantId); + ActionType.ASSIGNED_TO_TENANT, e, strTenantId); throw handleException(e); } } - private void pushSwappedFromNotification(Tenant currentTenant, TenantId newTenantId, Device swappedDevice) { - String data = entityToStr(swappedDevice); + private void pushAssignedFromNotification(Tenant currentTenant, TenantId newTenantId, Device assignedDevice) { + String data = entityToStr(assignedDevice); if (data != null) { - TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_SWAPPED_FROM, swappedDevice.getId(), getMetaDataForSwappedFrom(currentTenant), TbMsgDataType.JSON, data); - tbClusterService.pushMsgToRuleEngine(newTenantId, swappedDevice.getId(), tbMsg, null); + TbMsg tbMsg = TbMsg.newMsg(DataConstants.ENTITY_ASSIGNED_FROM_TENANT, assignedDevice.getId(), getMetaDataForAssignedFrom(currentTenant), TbMsgDataType.JSON, data); + tbClusterService.pushMsgToRuleEngine(newTenantId, assignedDevice.getId(), tbMsg, null); } } - private TbMsgMetaData getMetaDataForSwappedFrom(Tenant tenant) { + private TbMsgMetaData getMetaDataForAssignedFrom(Tenant tenant) { TbMsgMetaData metaData = new TbMsgMetaData(); - metaData.putValue("swappedFromTenantId", tenant.getId().getId().toString()); - metaData.putValue("swappedFromTenantName", tenant.getName()); + metaData.putValue("assignedFromTenantId", tenant.getId().getId().toString()); + metaData.putValue("assignedFromTenantName", tenant.getName()); return metaData; } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java index 0c4f62585d..2334c01102 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java +++ b/application/src/main/java/org/thingsboard/server/service/security/permission/Operation.java @@ -18,6 +18,6 @@ package org.thingsboard.server.service.security.permission; public enum Operation { ALL, CREATE, READ, WRITE, DELETE, ASSIGN_TO_CUSTOMER, UNASSIGN_FROM_CUSTOMER, RPC_CALL, - READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES + READ_CREDENTIALS, WRITE_CREDENTIALS, READ_ATTRIBUTES, WRITE_ATTRIBUTES, READ_TELEMETRY, WRITE_TELEMETRY, CLAIM_DEVICES, ASSIGN_TO_TENANT } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index 48cf397168..99de423086 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -800,7 +800,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { } @Test - public void testSwapDeviceFromOneTenantToAnother() throws Exception { + public void testAssignDeviceToTenant() throws Exception { Device device = new Device(); device.setName("My device"); device.setType("default"); @@ -816,7 +816,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { relation.setTo(savedAnotherDevice.getId()); relation.setTypeGroup(RelationTypeGroup.COMMON); relation.setType("Contains"); - doPost("/api/relation", relation); + doPost("/api/relation", relation).andExpect(status().isOk()); loginSysAdmin(); Tenant tenant = new Tenant(); @@ -834,13 +834,13 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { createUserAndLogin(user, "testPassword1"); login("tenant2@thingsboard.org", "testPassword1"); - Device swappedDevice = doPost("/api/tenant/" + savedDifferentTenant.getId().getId() + "/device/" + savedDevice.getId().getId(), Device.class); + Device assignedDevice = doPost("/api/tenant/" + savedDifferentTenant.getId().getId() + "/device/" + savedDevice.getId().getId(), Device.class); - doGet("/api/device/" + swappedDevice.getId().getId().toString(), Device.class, status().isNotFound()); + doGet("/api/device/" + assignedDevice.getId().getId().toString(), Device.class, status().isNotFound()); login("tenant9@thingsboard.org", "testPassword1"); - Device foundDevice1 = doGet("/api/device/" + swappedDevice.getId().getId().toString(), Device.class); + Device foundDevice1 = doGet("/api/device/" + assignedDevice.getId().getId().toString(), Device.class); Assert.assertNotNull(foundDevice1); doGet("/api/relation?fromId=" + savedDevice.getId().getId() + "&fromType=DEVICE&relationType=Contains&toId=" + savedAnotherDevice.getId().getId() + "&toType=DEVICE", EntityRelation.class, status().isNotFound()); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java index f54cda6681..ef6e62e931 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/audit/AuditLogService.java @@ -47,6 +47,4 @@ public interface AuditLogService { E entity, ActionType actionType, Exception e, Object... additionalInfo); - - void removeAuditLogs(TenantId tenantId, EntityId entityId); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java index 20516be837..1ed9ab9819 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/device/DeviceService.java @@ -65,6 +65,6 @@ public interface DeviceService { ListenableFuture> findDeviceTypesByTenantId(TenantId tenantId); - Device swapDevice(TenantId tenantId, Device device); + Device assignDeviceToTenant(TenantId tenantId, Device device); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java index 22b49552c0..e99db672a3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java @@ -57,8 +57,8 @@ public class DataConstants { public static final String ATTRIBUTES_DELETED = "ATTRIBUTES_DELETED"; public static final String ALARM_ACK = "ALARM_ACK"; public static final String ALARM_CLEAR = "ALARM_CLEAR"; - public static final String ENTITY_SWAPPED_FROM = "ENTITY_SWAPPED_FROM"; - public static final String ENTITY_SWAPPED_TO = "ENTITY_SWAPPED_TO"; + public static final String ENTITY_ASSIGNED_FROM_TENANT = "ENTITY_ASSIGNED_FROM_TENANT"; + public static final String ENTITY_ASSIGNED_TO_TENANT = "ENTITY_ASSIGNED_TO_TENANT"; public static final String RPC_CALL_FROM_SERVER_TO_DEVICE = "RPC_CALL_FROM_SERVER_TO_DEVICE"; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java index 70909dc331..14a38c810c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/audit/ActionType.java @@ -41,8 +41,8 @@ public enum ActionType { LOGIN(false), LOGOUT(false), LOCKOUT(false), - SWAPPED_FROM_TENANT(false), - SWAPPED_TO_TENANT(false); + ASSIGNED_FROM_TENANT(false), + ASSIGNED_TO_TENANT(false); private final boolean isRead; diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java index 0c9024af13..6a71045851 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/BaseAssetService.java @@ -159,7 +159,7 @@ public class BaseAssetService extends AbstractEntityService implements AssetServ try { List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(asset.getTenantId(), assetId).get(); if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete asset that is assigned to entity views!"); + throw new DataValidationException("Can't delete asset that has entity views!"); } } catch (ExecutionException | InterruptedException e) { log.error("Exception while finding entity views for assetId [{}]", assetId, e); diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java index 72b8272c69..0d1c708f68 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/AuditLogServiceImpl.java @@ -51,8 +51,6 @@ import org.thingsboard.server.dao.service.DataValidator; import java.io.PrintWriter; import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import static org.thingsboard.server.dao.service.Validator.validateEntityId; @@ -158,25 +156,6 @@ public class AuditLogServiceImpl implements AuditLogService { } } - @Override - public void removeAuditLogs(TenantId tenantId, EntityId entityId) { - List auditLogs = new ArrayList<>(); - TimePageData auditLogPageData; - TimePageLink auditLogPageLink = new TimePageLink(1000); - do { - auditLogPageData = findAuditLogsByTenantIdAndEntityId(tenantId, entityId, - new ArrayList<>(Arrays.asList(ActionType.values())), auditLogPageLink); - auditLogs.addAll(auditLogPageData.getData()); - if (auditLogPageData.hasNext()) { - auditLogPageLink = auditLogPageData.getNextPageLink(); - } - } while (auditLogPageData.hasNext()); - - for (AuditLog auditLog : auditLogs) { - auditLogDao.removeById(tenantId, auditLog.getUuidId()); - } - } - private JsonNode constructActionData(I entityId, E entity, ActionType actionType, Object... additionalInfo) { @@ -187,7 +166,7 @@ public class AuditLogServiceImpl implements AuditLogService { case ALARM_ACK: case ALARM_CLEAR: case RELATIONS_DELETED: - case SWAPPED_TO_TENANT: + case ASSIGNED_TO_TENANT: if (entity != null) { ObjectNode entityNode = objectMapper.valueToTree(entity); if (entityId.getEntityType() == EntityType.DASHBOARD) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java index c677b4368b..30e28a1bf2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/audit/DummyAuditLogServiceImpl.java @@ -58,8 +58,4 @@ public class DummyAuditLogServiceImpl implements AuditLogService { public ListenableFuture> logEntityAction(TenantId tenantId, CustomerId customerId, UserId userId, String userName, I entityId, E entity, ActionType actionType, Exception e, Object... additionalInfo) { return null; } - - @Override - public void removeAuditLogs(TenantId tenantId, EntityId entityId) { - } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 7abcaa73c3..cb6d74cf10 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -48,7 +48,6 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.EntitySearchDirection; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; -import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.entityview.EntityViewService; @@ -104,9 +103,6 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe @Autowired private EventService eventService; - @Autowired - private AuditLogService auditLogService; - @Override public Device findDeviceById(TenantId tenantId, DeviceId deviceId) { log.trace("Executing findDeviceById [{}]", deviceId); @@ -201,7 +197,7 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe try { List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), deviceId).get(); if (entityViews != null && !entityViews.isEmpty()) { - throw new DataValidationException("Can't delete device that is assigned to entity views!"); + throw new DataValidationException("Can't delete device that has entity views!"); } } catch (ExecutionException | InterruptedException e) { log.error("Exception while finding entity views for deviceId [{}]", deviceId, e); @@ -338,13 +334,13 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe @Transactional @CacheEvict(cacheNames = DEVICE_CACHE, key = "{#device.tenantId, #device.name}") @Override - public Device swapDevice(TenantId tenantId, Device device) { - log.trace("Executing swapDevice [{}]", device); + public Device assignDeviceToTenant(TenantId tenantId, Device device) { + log.trace("Executing assignDeviceToTenant [{}][{}]", tenantId, device); try { List entityViews = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(device.getTenantId(), device.getId()).get(); if (!CollectionUtils.isEmpty(entityViews)) { - throw new DataValidationException("Can't swap device that is assigned to entity views!"); + throw new DataValidationException("Can't assign device that has entity views to another tenant!"); } } catch (ExecutionException | InterruptedException e) { log.error("Exception while finding entity views for deviceId [{}]", device.getId(), e); @@ -355,11 +351,6 @@ public class DeviceServiceImpl extends AbstractEntityService implements DeviceSe relationService.removeRelations(device.getTenantId(), device.getId()); - // TODO: 30/07/2020 implement for Cassandra - if (sqlDatabaseUsed) { - auditLogService.removeAuditLogs(device.getTenantId(), device.getId()); - } - device.setTenantId(tenantId); device.setCustomerId(null); return doSaveDevice(device, null); diff --git a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java index 50d49e75ad..88e701d3bd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/event/BaseEventService.java @@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; -import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -97,20 +96,17 @@ public class BaseEventService implements EventService { @Override public void removeEvents(TenantId tenantId, EntityId entityId) { - List events = new ArrayList<>(); TimePageData eventPageData; TimePageLink eventPageLink = new TimePageLink(1000); do { eventPageData = findEvents(tenantId, entityId, eventPageLink); - events.addAll(eventPageData.getData()); + for (Event event : eventPageData.getData()) { + eventDao.removeById(tenantId, event.getUuidId()); + } if (eventPageData.hasNext()) { eventPageLink = eventPageData.getNextPageLink(); } } while (eventPageData.hasNext()); - - for (Event event : events) { - eventDao.removeById(tenantId, event.getUuidId()); - } } private DataValidator eventValidator = diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java index 62e75cd72a..c5739e24ff 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/filter/TbMsgTypeSwitchNode.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.msg.session.SessionMsgType; configClazz = EmptyNodeConfiguration.class, relationTypes = {"Post attributes", "Post telemetry", "RPC Request from Device", "RPC Request to Device", "Activity Event", "Inactivity Event", "Connect Event", "Disconnect Event", "Entity Created", "Entity Updated", "Entity Deleted", "Entity Assigned", - "Entity Unassigned", "Attributes Updated", "Attributes Deleted", "Alarm Acknowledged", "Alarm Cleared", "Other", "Entity Swapped From", "Entity Swapped To"}, + "Entity Unassigned", "Attributes Updated", "Attributes Deleted", "Alarm Acknowledged", "Alarm Cleared", "Other", "Entity Assigned From Tenant", "Entity Assigned To Tenant"}, nodeDescription = "Route incoming messages by Message Type", nodeDetails = "Sends messages with message types \"Post attributes\", \"Post telemetry\", \"RPC Request\" etc. via corresponding chain, otherwise Other chain is used.", uiResources = {"static/rulenode/rulenode-core-config.js"}, @@ -86,10 +86,10 @@ public class TbMsgTypeSwitchNode implements TbNode { relationType = "Alarm Cleared"; } else if (msg.getType().equals(DataConstants.RPC_CALL_FROM_SERVER_TO_DEVICE)) { relationType = "RPC Request to Device"; - } else if (msg.getType().equals(DataConstants.ENTITY_SWAPPED_FROM)) { - relationType = "Entity Swapped From"; - } else if (msg.getType().equals(DataConstants.ENTITY_SWAPPED_TO)) { - relationType = "Entity Swapped To"; + } else if (msg.getType().equals(DataConstants.ENTITY_ASSIGNED_FROM_TENANT)) { + relationType = "Entity Assigned From Tenant"; + } else if (msg.getType().equals(DataConstants.ENTITY_ASSIGNED_TO_TENANT)) { + relationType = "Entity Assigned To Tenant"; } else { relationType = "Other"; } diff --git a/ui/src/app/common/types.constant.js b/ui/src/app/common/types.constant.js index 254313b7a0..da2913bdbd 100644 --- a/ui/src/app/common/types.constant.js +++ b/ui/src/app/common/types.constant.js @@ -223,11 +223,11 @@ export default angular.module('thingsboard.types', []) "LOCKOUT": { name: "audit-log.type-lockout" }, - "SWAPPED_FROM_TENANT": { - name: "audit-log.type-swapped-from-tenant" + "ASSIGNED_FROM_TENANT": { + name: "audit-log.type-assigned-from-tenant" }, - "SWAPPED_TO_TENANT": { - name: "audit-log.type-swapped-to-tenant" + "ASSIGNED_TO_TENANT": { + name: "audit-log.type-assigned-to-tenant" } }, auditLogActionStatus: { diff --git a/ui/src/app/locale/locale.constant-en_US.json b/ui/src/app/locale/locale.constant-en_US.json index 7e81fd1a57..2b21ec44be 100644 --- a/ui/src/app/locale/locale.constant-en_US.json +++ b/ui/src/app/locale/locale.constant-en_US.json @@ -357,8 +357,8 @@ "failure-details": "Failure details", "search": "Search audit logs", "clear-search": "Clear search", - "type-swapped-from-tenant": "Swapped from Tenant", - "type-swapped-to-tenant": "Swapped to Tenant" + "type-assigned-from-tenant": "Assigned from Tenant", + "type-assigned-to-tenant": "Assigned to Tenant" }, "confirm-on-exit": { "message": "You have unsaved changes. Are you sure you want to leave this page?", From 2b95d6cbaed06d6c1f752abb4b728ceffa61ac23 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Fri, 31 Jul 2020 15:06:47 +0300 Subject: [PATCH 07/39] changed logic to report activity & improvements for clear alarm node --- .../state/DefaultDeviceStateService.java | 8 ++--- .../engine/action/TbAbstractAlarmNode.java | 2 +- .../rule/engine/action/TbClearAlarmNode.java | 36 ++++++++++++++++--- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 8acca0b4c7..f8f0c6040c 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -197,15 +197,15 @@ public class DefaultDeviceStateService implements DeviceStateService { if (lastReportedActivity > 0 && lastReportedActivity > lastSavedActivity) { DeviceStateData stateData = getOrFetchDeviceStateData(deviceId); if (stateData != null) { - DeviceState state = stateData.getState(); - stateData.getState().setLastActivityTime(lastReportedActivity); - stateData.getMetaData().putValue("scope", SERVER_SCOPE); - pushRuleEngineMessage(stateData, ACTIVITY_EVENT); save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); deviceLastSavedActivity.put(deviceId, lastReportedActivity); + DeviceState state = stateData.getState(); if (!state.isActive()) { state.setActive(true); save(deviceId, ACTIVITY_STATE, state.isActive()); + state.setLastActivityTime(lastReportedActivity); + stateData.getMetaData().putValue("scope", SERVER_SCOPE); + pushRuleEngineMessage(stateData, ACTIVITY_EVENT); } } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index b3344105bb..766522d207 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -41,7 +41,7 @@ public abstract class TbAbstractAlarmNode processAlarm(TbContext ctx, TbMsg msg) { String alarmType = TbNodeUtils.processPattern(this.config.getAlarmType(), msg.getMetaData()); - ListenableFuture latest = ctx.getAlarmService().findLatestByOriginatorAndType(ctx.getTenantId(), msg.getOriginator(), alarmType); - return Futures.transformAsync(latest, a -> { - if (a != null && !a.getStatus().isCleared()) { - return clearAlarm(ctx, msg, a); + if (msg.getOriginator().getEntityType().equals(EntityType.ALARM)) { + return clearAlarmFromOriginator(ctx, msg); + } else { + ListenableFuture latest = ctx.getAlarmService().findLatestByOriginatorAndType(ctx.getTenantId(), msg.getOriginator(), alarmType); + return Futures.transformAsync(latest, a -> { + if (a != null && !a.getStatus().isCleared()) { + return clearAlarm(ctx, msg, a); + } + return Futures.immediateFuture(new AlarmResult(false, false, false, null)); + }, ctx.getDbCallbackExecutor()); + } + } + + private ListenableFuture clearAlarmFromOriginator(TbContext ctx, TbMsg msg) { + ListenableFuture alarmByIdAsync = ctx.getAlarmService().findAlarmByIdAsync(ctx.getTenantId(), new AlarmId(msg.getOriginator().getId())); + return Futures.transformAsync(alarmByIdAsync, alarm -> { + if (alarm != null && !alarm.getStatus().isCleared()) { + long clearTs = System.currentTimeMillis(); + ListenableFuture clearAlarmFuture = ctx.getAlarmService().clearAlarm(ctx.getTenantId(), alarm.getId(), alarm.getDetails(), clearTs); + return Futures.transformAsync(clearAlarmFuture, cleared -> { + if (cleared) { + alarm.setClearTs(clearTs); + AlarmStatus oldStatus = alarm.getStatus(); + AlarmStatus newStatus = oldStatus.isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK; + alarm.setStatus(newStatus); + return Futures.immediateFuture(new AlarmResult(false, false, true, alarm)); + } + return Futures.immediateFuture(new AlarmResult(false, false, false, alarm)); + }, ctx.getDbCallbackExecutor()); } return Futures.immediateFuture(new AlarmResult(false, false, false, null)); }, ctx.getDbCallbackExecutor()); From addf0ede2789be19a53a51f0816ce6c56fe42c92 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Fri, 31 Jul 2020 16:29:49 +0300 Subject: [PATCH 08/39] added buildAlarmDetails for clearAlarmByAlarmOriginator method --- .../rule/engine/action/TbClearAlarmNode.java | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java index a7d3c50e7a..d3c6429e34 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java @@ -58,7 +58,7 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode processAlarm(TbContext ctx, TbMsg msg) { String alarmType = TbNodeUtils.processPattern(this.config.getAlarmType(), msg.getMetaData()); if (msg.getOriginator().getEntityType().equals(EntityType.ALARM)) { - return clearAlarmFromOriginator(ctx, msg); + return clearAlarmByAlarmOriginator(ctx, msg); } else { ListenableFuture latest = ctx.getAlarmService().findLatestByOriginatorAndType(ctx.getTenantId(), msg.getOriginator(), alarmType); return Futures.transformAsync(latest, a -> { @@ -70,21 +70,24 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode clearAlarmFromOriginator(TbContext ctx, TbMsg msg) { + private ListenableFuture clearAlarmByAlarmOriginator(TbContext ctx, TbMsg msg) { ListenableFuture alarmByIdAsync = ctx.getAlarmService().findAlarmByIdAsync(ctx.getTenantId(), new AlarmId(msg.getOriginator().getId())); return Futures.transformAsync(alarmByIdAsync, alarm -> { if (alarm != null && !alarm.getStatus().isCleared()) { - long clearTs = System.currentTimeMillis(); - ListenableFuture clearAlarmFuture = ctx.getAlarmService().clearAlarm(ctx.getTenantId(), alarm.getId(), alarm.getDetails(), clearTs); - return Futures.transformAsync(clearAlarmFuture, cleared -> { - if (cleared) { - alarm.setClearTs(clearTs); - AlarmStatus oldStatus = alarm.getStatus(); - AlarmStatus newStatus = oldStatus.isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK; - alarm.setStatus(newStatus); + ctx.logJsEvalRequest(); + ListenableFuture asyncDetails = buildAlarmDetails(ctx, msg, alarm.getDetails()); + return Futures.transformAsync(asyncDetails, details -> { + ctx.logJsEvalRequest(); + long clearTs = System.currentTimeMillis(); + ListenableFuture clearAlarmFuture = ctx.getAlarmService().clearAlarm(ctx.getTenantId(), alarm.getId(), details, clearTs); + return Futures.transformAsync(clearAlarmFuture, cleared -> { + if (cleared) { + alarm.setClearTs(clearTs); + alarm.setDetails(details); + } + alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK); return Futures.immediateFuture(new AlarmResult(false, false, true, alarm)); - } - return Futures.immediateFuture(new AlarmResult(false, false, false, alarm)); + }, ctx.getDbCallbackExecutor()); }, ctx.getDbCallbackExecutor()); } return Futures.immediateFuture(new AlarmResult(false, false, false, null)); From 158436b14147709c8c7544e1a4763687b10f3d8f Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Fri, 31 Jul 2020 16:49:24 +0300 Subject: [PATCH 09/39] fix typo --- .../server/service/state/DefaultDeviceStateService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index f8f0c6040c..0025f80d71 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -200,10 +200,10 @@ public class DefaultDeviceStateService implements DeviceStateService { save(deviceId, LAST_ACTIVITY_TIME, lastReportedActivity); deviceLastSavedActivity.put(deviceId, lastReportedActivity); DeviceState state = stateData.getState(); + state.setLastActivityTime(lastReportedActivity); if (!state.isActive()) { state.setActive(true); save(deviceId, ACTIVITY_STATE, state.isActive()); - state.setLastActivityTime(lastReportedActivity); stateData.getMetaData().putValue("scope", SERVER_SCOPE); pushRuleEngineMessage(stateData, ACTIVITY_EVENT); } From 26e84f7b1834955648ab54acd4ea7bf15941282e Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Mon, 3 Aug 2020 13:54:40 +0300 Subject: [PATCH 10/39] refactoring & added alarmCanBeClearedWithAlarmOriginator test --- .../rule/engine/action/TbClearAlarmNode.java | 35 +++---------- .../rule/engine/action/TbAlarmNodeTest.java | 51 +++++++++++++++++++ 2 files changed, 57 insertions(+), 29 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java index d3c6429e34..30bf2d4d6e 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbClearAlarmNode.java @@ -57,38 +57,15 @@ public class TbClearAlarmNode extends TbAbstractAlarmNode processAlarm(TbContext ctx, TbMsg msg) { String alarmType = TbNodeUtils.processPattern(this.config.getAlarmType(), msg.getMetaData()); + ListenableFuture alarmFuture; if (msg.getOriginator().getEntityType().equals(EntityType.ALARM)) { - return clearAlarmByAlarmOriginator(ctx, msg); + alarmFuture = ctx.getAlarmService().findAlarmByIdAsync(ctx.getTenantId(), new AlarmId(msg.getOriginator().getId())); } else { - ListenableFuture latest = ctx.getAlarmService().findLatestByOriginatorAndType(ctx.getTenantId(), msg.getOriginator(), alarmType); - return Futures.transformAsync(latest, a -> { - if (a != null && !a.getStatus().isCleared()) { - return clearAlarm(ctx, msg, a); - } - return Futures.immediateFuture(new AlarmResult(false, false, false, null)); - }, ctx.getDbCallbackExecutor()); + alarmFuture = ctx.getAlarmService().findLatestByOriginatorAndType(ctx.getTenantId(), msg.getOriginator(), alarmType); } - } - - private ListenableFuture clearAlarmByAlarmOriginator(TbContext ctx, TbMsg msg) { - ListenableFuture alarmByIdAsync = ctx.getAlarmService().findAlarmByIdAsync(ctx.getTenantId(), new AlarmId(msg.getOriginator().getId())); - return Futures.transformAsync(alarmByIdAsync, alarm -> { - if (alarm != null && !alarm.getStatus().isCleared()) { - ctx.logJsEvalRequest(); - ListenableFuture asyncDetails = buildAlarmDetails(ctx, msg, alarm.getDetails()); - return Futures.transformAsync(asyncDetails, details -> { - ctx.logJsEvalRequest(); - long clearTs = System.currentTimeMillis(); - ListenableFuture clearAlarmFuture = ctx.getAlarmService().clearAlarm(ctx.getTenantId(), alarm.getId(), details, clearTs); - return Futures.transformAsync(clearAlarmFuture, cleared -> { - if (cleared) { - alarm.setClearTs(clearTs); - alarm.setDetails(details); - } - alarm.setStatus(alarm.getStatus().isAck() ? AlarmStatus.CLEARED_ACK : AlarmStatus.CLEARED_UNACK); - return Futures.immediateFuture(new AlarmResult(false, false, true, alarm)); - }, ctx.getDbCallbackExecutor()); - }, ctx.getDbCallbackExecutor()); + return Futures.transformAsync(alarmFuture, a -> { + if (a != null && !a.getStatus().isCleared()) { + return clearAlarm(ctx, msg, a); } return Futures.immediateFuture(new AlarmResult(false, false, false, null)); }, ctx.getDbCallbackExecutor()); diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java index 61b83d647d..114967eb51 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/action/TbAlarmNodeTest.java @@ -35,6 +35,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.data.alarm.Alarm; +import org.thingsboard.server.common.data.id.AlarmId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.RuleChainId; @@ -95,6 +96,7 @@ public class TbAlarmNodeTest { private ListeningExecutor dbExecutor; private EntityId originator = new DeviceId(UUIDs.timeBased()); + private EntityId alarmOriginator = new AlarmId(UUIDs.timeBased()); private TenantId tenantId = new TenantId(UUIDs.timeBased()); private TbMsgMetaData metaData = new TbMsgMetaData(); private String rawJson = "{\"name\": \"Vit\", \"passed\": 5}"; @@ -325,6 +327,55 @@ public class TbAlarmNodeTest { assertEquals(expectedAlarm, actualAlarm); } + @Test + public void alarmCanBeClearedWithAlarmOriginator() throws ScriptException, IOException { + initWithClearAlarmScript(); + metaData.putValue("key", "value"); + TbMsg msg = TbMsg.newMsg( "USER", alarmOriginator, metaData, TbMsgDataType.JSON, rawJson, ruleChainId, ruleNodeId); + + long oldEndDate = System.currentTimeMillis(); + AlarmId id = new AlarmId(alarmOriginator.getId()); + Alarm activeAlarm = Alarm.builder().type("SomeType").tenantId(tenantId).originator(originator).status(ACTIVE_UNACK).severity(WARNING).endTs(oldEndDate).build(); + activeAlarm.setId(id); + + when(detailsJs.executeJsonAsync(msg)).thenReturn(Futures.immediateFuture(null)); + when(alarmService.findAlarmByIdAsync(tenantId, id)).thenReturn(Futures.immediateFuture(activeAlarm)); + when(alarmService.clearAlarm(eq(activeAlarm.getTenantId()), eq(activeAlarm.getId()), org.mockito.Mockito.any(JsonNode.class), anyLong())).thenReturn(Futures.immediateFuture(true)); +// doAnswer((Answer) invocationOnMock -> (Alarm) (invocationOnMock.getArguments())[0]).when(alarmService).createOrUpdateAlarm(activeAlarm); + + node.onMsg(ctx, msg); + + verify(ctx).tellNext(any(), eq("Cleared")); + + ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(TbMsg.class); + ArgumentCaptor typeCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor originatorCaptor = ArgumentCaptor.forClass(EntityId.class); + ArgumentCaptor metadataCaptor = ArgumentCaptor.forClass(TbMsgMetaData.class); + ArgumentCaptor dataCaptor = ArgumentCaptor.forClass(String.class); + verify(ctx).transformMsg(msgCaptor.capture(), typeCaptor.capture(), originatorCaptor.capture(), metadataCaptor.capture(), dataCaptor.capture()); + + assertEquals("ALARM", typeCaptor.getValue()); + assertEquals(alarmOriginator, originatorCaptor.getValue()); + assertEquals("value", metadataCaptor.getValue().getValue("key")); + assertEquals(Boolean.TRUE.toString(), metadataCaptor.getValue().getValue(IS_CLEARED_ALARM)); + assertNotSame(metaData, metadataCaptor.getValue()); + + Alarm actualAlarm = new ObjectMapper().readValue(dataCaptor.getValue().getBytes(), Alarm.class); + Alarm expectedAlarm = Alarm.builder() + .tenantId(tenantId) + .originator(originator) + .status(CLEARED_UNACK) + .severity(WARNING) + .propagate(false) + .type("SomeType") + .details(null) + .endTs(oldEndDate) + .build(); + expectedAlarm.setId(id); + + assertEquals(expectedAlarm, actualAlarm); + } + private void initWithCreateAlarmScript() { try { TbCreateAlarmNodeConfiguration config = new TbCreateAlarmNodeConfiguration(); From 45e3c2297117ae29f800a4376f49c4a721f1a907 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Mon, 3 Aug 2020 13:57:45 +0300 Subject: [PATCH 11/39] fix typo --- .../org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java index 766522d207..b3344105bb 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/action/TbAbstractAlarmNode.java @@ -41,7 +41,7 @@ public abstract class TbAbstractAlarmNode Date: Thu, 18 Jun 2020 15:27:11 +0200 Subject: [PATCH 12/39] enable default credential provider chain for aws sqs --- application/src/main/resources/thingsboard.yml | 4 ++++ .../server/queue/sqs/TbAwsSqsAdmin.java | 15 ++++++++++----- .../server/queue/sqs/TbAwsSqsSettings.java | 3 +++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cde049b99f..f4d76be201 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -612,6 +612,10 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: + # @see https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-roles.html + # setting this to true, will ignore the access keys below and instead use the + # default credential provider chain, which includes instance profile credentials etc. + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java index 8e293e6dff..7c13e32740 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.queue.sqs; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.*; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.CreateQueueRequest; @@ -37,9 +35,16 @@ public class TbAwsSqsAdmin implements TbQueueAdmin { public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map attributes) { this.attributes = attributes; - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + AWSCredentialsProvider credentialsProvider; + if (sqsSettings.getUseDefaultCredentialProviderChain()) { + credentialsProvider = new DefaultAWSCredentialsProviderChain(); + } else { + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials); + } + sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(awsCredentials)) + .withCredentials(credentialsProvider) .withRegion(sqsSettings.getRegion()) .build(); diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java index 922a2b1062..7a5c3332ad 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java @@ -27,6 +27,9 @@ import org.springframework.stereotype.Component; @Data public class TbAwsSqsSettings { + @Value("${queue.aws_sqs.use_default_credential_provider_chain}") + private Boolean useDefaultCredentialProviderChain; + @Value("${queue.aws_sqs.access_key_id}") private String accessKeyId; From cc79b026035e6b9711fd142405b06bf1fcae88ad Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 3 Aug 2020 14:57:07 +0300 Subject: [PATCH 13/39] added default credentials provider chain for aws sqs consumer and producer --- .../src/main/resources/thingsboard.yml | 3 --- .../server/queue/sqs/TbAwsSqsAdmin.java | 6 +++++- .../queue/sqs/TbAwsSqsConsumerTemplate.java | 16 ++++++++++++---- .../queue/sqs/TbAwsSqsProducerTemplate.java | 19 +++++++++++-------- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f4d76be201..73db8c6735 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -612,9 +612,6 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: - # @see https://docs.aws.amazon.com/sdk-for-java/v1/developer-guide/java-dg-roles.html - # setting this to true, will ignore the access keys below and instead use the - # default credential provider chain, which includes instance profile credentials etc. use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java index 7c13e32740..f99755a0af 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java @@ -15,7 +15,11 @@ */ package org.thingsboard.server.queue.sqs; -import com.amazonaws.auth.*; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.CreateQueueRequest; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java index f4f279a02b..3e30123980 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsConsumerTemplate.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.sqs; import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.DeleteMessageBatchRequestEntry; @@ -67,13 +69,19 @@ public class TbAwsSqsConsumerTemplate extends AbstractPara this.decoder = decoder; this.sqsSettings = sqsSettings; - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + AWSCredentialsProvider credentialsProvider; + if (sqsSettings.getUseDefaultCredentialProviderChain()) { + credentialsProvider = new DefaultAWSCredentialsProviderChain(); + } else { + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials); + } - this.sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) + sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credentialsProvider) .withRegion(sqsSettings.getRegion()) .build(); + } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 6110d08c5e..79bc6c803a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -16,8 +16,10 @@ package org.thingsboard.server.queue.sqs; import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; import com.amazonaws.services.sqs.model.SendMessageRequest; @@ -26,7 +28,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -39,7 +40,6 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; @Slf4j public class TbAwsSqsProducerTemplate implements TbQueueProducer { @@ -54,15 +54,18 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr this.admin = admin; this.defaultTopic = defaultTopic; - AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); - AWSStaticCredentialsProvider credProvider = new AWSStaticCredentialsProvider(awsCredentials); + AWSCredentialsProvider credentialsProvider; + if (sqsSettings.getUseDefaultCredentialProviderChain()) { + credentialsProvider = new DefaultAWSCredentialsProviderChain(); + } else { + AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey()); + credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials); + } - this.sqsClient = AmazonSQSClientBuilder.standard() - .withCredentials(credProvider) + sqsClient = AmazonSQSClientBuilder.standard() + .withCredentials(credentialsProvider) .withRegion(sqsSettings.getRegion()) .build(); - - producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); } @Override From 090c36999f97a6427f308770d1bd7cf75b751309 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 3 Aug 2020 15:47:18 +0300 Subject: [PATCH 14/39] refactored --- .../queue/sqs/TbAwsSqsProducerTemplate.java | 3 +++ msa/js-executor/package-lock.json | 23 ++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java index 79bc6c803a..3e508b09e6 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.Gson; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -40,6 +41,7 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsg; import java.util.Map; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; @Slf4j public class TbAwsSqsProducerTemplate implements TbQueueProducer { @@ -66,6 +68,7 @@ public class TbAwsSqsProducerTemplate implements TbQueuePr .withCredentials(credentialsProvider) .withRegion(sqsSettings.getRegion()) .build(); + producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool()); } @Override diff --git a/msa/js-executor/package-lock.json b/msa/js-executor/package-lock.json index 5636c4f586..47d7dba886 100644 --- a/msa/js-executor/package-lock.json +++ b/msa/js-executor/package-lock.json @@ -1872,12 +1872,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1892,17 +1894,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -2019,7 +2024,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -2031,6 +2037,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -2045,6 +2052,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -2156,7 +2164,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -2168,6 +2177,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -2289,6 +2299,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", From a8fe8b9e10a8a0d5bf18b47e1f529c60c671b472 Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Wed, 5 Aug 2020 13:08:40 +0300 Subject: [PATCH 15/39] fix typo in ConditionalOnExpression annotation --- .../server/transport/mqtt/MqttSslHandlerProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java index 4e3a39699a..0c872c351f 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttSslHandlerProvider.java @@ -53,7 +53,7 @@ import java.util.concurrent.TimeUnit; */ @Slf4j @Component("MqttSslHandlerProvider") -@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.http.enabled}'=='true')") +@ConditionalOnExpression("'${transport.type:null}'=='null' || ('${transport.type}'=='local' && '${transport.mqtt.enabled}'=='true')") @ConditionalOnProperty(prefix = "transport.mqtt.ssl", value = "enabled", havingValue = "true", matchIfMissing = false) public class MqttSslHandlerProvider { From 86102aea06735fcd7f7f5d306a55817eb2b23d34 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 4 Aug 2020 12:38:58 +0300 Subject: [PATCH 16/39] added other parameters for queue kafka --- application/src/main/resources/thingsboard.yml | 10 ++++++++++ .../server/queue/kafka/TbKafkaSettings.java | 5 ++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 73db8c6735..2a58209560 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -605,6 +605,16 @@ queue: max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + other: +# Properties for Confluent cloud +# - key: "ssl.endpoint.identification.algorithm" +# value: "https" +# - key: "sasl.mechanism" +# value: "PLAIN" +# - key: "sasl.jaas.config" +# value: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";" +# - key: "security.protocol" +# value: "SASL_SSL" topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 659dd19bda..d9da969eb9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -16,10 +16,12 @@ package org.thingsboard.server.queue.kafka; import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import java.util.List; @@ -30,6 +32,7 @@ import java.util.Properties; */ @Slf4j @ConditionalOnExpression("'${queue.type:null}'=='kafka'") +@ConfigurationProperties(prefix = "queue.kafka") @Component public class TbKafkaSettings { @@ -67,7 +70,7 @@ public class TbKafkaSettings { @Getter private int fetchMaxBytes; - @Value("${kafka.other:#{null}}") + @Setter private List other; public Properties toProps() { From b445473e3cf0ea2ee138dcdd85d7e74d3af6eed1 Mon Sep 17 00:00:00 2001 From: nordmif Date: Thu, 9 Jul 2020 14:22:38 +0300 Subject: [PATCH 17/39] added logging of MQTT payload errors --- .../server/transport/mqtt/adaptors/JsonMqttAdaptor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index 45370a2c89..51a4c359f6 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -207,6 +207,7 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { try { return new JsonParser().parse(payload); } catch (JsonSyntaxException ex) { + log.error(payload); throw new AdaptorException(ex); } } From 045c8e3acaaaa38f61bb75c1997ac1c112e0db55 Mon Sep 17 00:00:00 2001 From: nordmif Date: Thu, 9 Jul 2020 14:27:08 +0300 Subject: [PATCH 18/39] added logging of MQTT payload errors --- .../server/transport/mqtt/adaptors/JsonMqttAdaptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index 51a4c359f6..755ebc9334 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -207,7 +207,7 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { try { return new JsonParser().parse(payload); } catch (JsonSyntaxException ex) { - log.error(payload); + log.error("Payload is in incorrect format: " + payload); throw new AdaptorException(ex); } } From e5a6ddb03b0eba03d76dca08ede14cec071cee28 Mon Sep 17 00:00:00 2001 From: nordmif Date: Thu, 9 Jul 2020 14:56:09 +0300 Subject: [PATCH 19/39] added logging of MQTT payload errors --- .../server/transport/mqtt/adaptors/JsonMqttAdaptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index 755ebc9334..6a28d9b47e 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -207,7 +207,7 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { try { return new JsonParser().parse(payload); } catch (JsonSyntaxException ex) { - log.error("Payload is in incorrect format: " + payload); + log.error("Payload is in incorrect format: {}", payload); throw new AdaptorException(ex); } } From 52bed1748958ecf97bd33470875dc444cbcadb6d Mon Sep 17 00:00:00 2001 From: nordmif Date: Tue, 21 Jul 2020 15:03:45 +0300 Subject: [PATCH 20/39] changed logging level to WARN --- .../server/transport/mqtt/adaptors/JsonMqttAdaptor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index 6a28d9b47e..caa627451f 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -207,7 +207,7 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { try { return new JsonParser().parse(payload); } catch (JsonSyntaxException ex) { - log.error("Payload is in incorrect format: {}", payload); + log.warn("Payload is in incorrect format: {}", payload); throw new AdaptorException(ex); } } From 87b23f4989062595509fb1820a1906fce5a8aae5 Mon Sep 17 00:00:00 2001 From: Valerii Sosliuk Date: Mon, 10 Aug 2020 23:34:52 +0300 Subject: [PATCH 21/39] Display sorted metadata keys in rule nodes --- ui/src/app/common/utils.service.js | 13 ++++++++++++- ui/src/app/event/event-row.directive.js | 11 +++++++++-- .../rulechain/script/node-script-test.service.js | 4 +++- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ui/src/app/common/utils.service.js b/ui/src/app/common/utils.service.js index 97a04bbcf2..d5e4852afd 100644 --- a/ui/src/app/common/utils.service.js +++ b/ui/src/app/common/utils.service.js @@ -150,7 +150,8 @@ function Utils($mdColorPalette, $rootScope, $window, $translate, $q, $timeout, t customTranslation: customTranslation, objToBase64: objToBase64, base64toObj: base64toObj, - loadImageAspect: loadImageAspect + loadImageAspect: loadImageAspect, + sortObjectKeys: sortObjectKeys } return service; @@ -605,4 +606,14 @@ function Utils($mdColorPalette, $rootScope, $window, $translate, $q, $timeout, t return deferred.promise; } + function sortObjectKeys(obj) { + var sortedObj = {}; + var keys = Object.keys(obj).sort(); + for (var i = 0; i < keys.length; i++) { + var key = keys[i]; + sortedObj[key] = obj[key]; + } + return sortedObj; + } + } diff --git a/ui/src/app/event/event-row.directive.js b/ui/src/app/event/event-row.directive.js index eec8a1aff6..ffd5cc4adf 100644 --- a/ui/src/app/event/event-row.directive.js +++ b/ui/src/app/event/event-row.directive.js @@ -25,7 +25,7 @@ import eventRowDebugRuleNodeTemplate from './event-row-debug-rulenode.tpl.html'; /* eslint-enable import/no-unresolved, import/default */ /*@ngInject*/ -export default function EventRowDirective($compile, $templateCache, $mdDialog, $document, types) { +export default function EventRowDirective($compile, $templateCache, $mdDialog, $document, types, utils) { var linker = function (scope, element, attrs) { @@ -71,11 +71,18 @@ export default function EventRowDirective($compile, $templateCache, $mdDialog, $ if (!contentType) { contentType = null; } + var sortedContent; + try { + sortedContent = angular.toJson(utils.sortObjectKeys(angular.fromJson(content))); + } + catch(err) { + sortedContent = content; + } $mdDialog.show({ controller: 'EventContentDialogController', controllerAs: 'vm', templateUrl: eventErrorDialogTemplate, - locals: {content: content, title: title, contentType: contentType, showingCallback: onShowingCallback}, + locals: {content: sortedContent, title: title, contentType: contentType, showingCallback: onShowingCallback}, parent: angular.element($document[0].body), fullscreen: true, targetEvent: $event, diff --git a/ui/src/app/rulechain/script/node-script-test.service.js b/ui/src/app/rulechain/script/node-script-test.service.js index ee6eea8378..ae1f0b0568 100644 --- a/ui/src/app/rulechain/script/node-script-test.service.js +++ b/ui/src/app/rulechain/script/node-script-test.service.js @@ -20,7 +20,7 @@ import nodeScriptTestTemplate from './node-script-test.tpl.html'; /* eslint-enable import/no-unresolved, import/default */ /*@ngInject*/ -export default function NodeScriptTest($q, $mdDialog, $document, ruleChainService) { +export default function NodeScriptTest($q, $mdDialog, $document, ruleChainService, utils) { var service = { testNodeScript: testNodeScript @@ -89,6 +89,8 @@ export default function NodeScriptTest($q, $mdDialog, $document, ruleChainServic deviceName: "Test Device", ts: new Date().getTime() + "" }; + } else { + metadata = utils.sortObjectKeys(metadata); } if (!msgType) { msgType = "POST_TELEMETRY_REQUEST"; From e318b193bd604eaa1e124d3f44f438838e8b20a6 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Tue, 11 Aug 2020 11:15:41 +0300 Subject: [PATCH 22/39] Develop/2.5.3 confluent cloud (#3259) * added other parameters for queue kafka * Added support Confluent Cloud * fix js executor kafka connection * refactored --- .../src/main/resources/thingsboard.yml | 15 ++--- .../server/queue/kafka/TbKafkaSettings.java | 37 ++++++++++-- docker/compose-utils.sh | 5 +- docker/docker-compose.confluent.yml | 57 +++++++++++++++++++ docker/queue-confluent.env | 18 ++++++ .../config/custom-environment-variables.yml | 6 ++ msa/js-executor/config/default.yml | 4 ++ msa/js-executor/queue/kafkaTemplate.js | 20 +++++-- .../src/main/resources/tb-coap-transport.yml | 8 +++ .../src/main/resources/tb-http-transport.yml | 8 +++ .../src/main/resources/tb-mqtt-transport.yml | 8 +++ 11 files changed, 166 insertions(+), 20 deletions(-) create mode 100644 docker/docker-compose.confluent.yml create mode 100644 docker/queue-confluent.env diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2a58209560..b6894bb0e7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -605,16 +605,13 @@ queue: max_poll_records: "${TB_QUEUE_KAFKA_MAX_POLL_RECORDS:8192}" max_partition_fetch_bytes: "${TB_QUEUE_KAFKA_MAX_PARTITION_FETCH_BYTES:16777216}" fetch_max_bytes: "${TB_QUEUE_KAFKA_FETCH_MAX_BYTES:134217728}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: -# Properties for Confluent cloud -# - key: "ssl.endpoint.identification.algorithm" -# value: "https" -# - key: "sasl.mechanism" -# value: "PLAIN" -# - key: "sasl.jaas.config" -# value: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";" -# - key: "security.protocol" -# value: "SASL_SSL" topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index d9da969eb9..37e978c07d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -18,6 +18,7 @@ package org.thingsboard.server.queue.kafka; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.clients.producer.ProducerConfig; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; @@ -68,7 +69,22 @@ public class TbKafkaSettings { @Value("${queue.kafka.fetch_max_bytes:134217728}") @Getter - private int fetchMaxBytes; + private int fetchMaxBytes; + + @Value("${queue.kafka.use_confluent_cloud:false}") + private boolean useConfluent; + + @Value("${queue.kafka.confluent.ssl.algorithm}") + private String sslAlgorithm; + + @Value("${queue.kafka.confluent.sasl.mechanism}") + private String saslMechanism; + + @Value("${queue.kafka.confluent.sasl.config}") + private String saslConfig; + + @Value("${queue.kafka.confluent.security.protocol}") + private String securityProtocol; @Setter private List other; @@ -76,12 +92,21 @@ public class TbKafkaSettings { public Properties toProps() { Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, servers); - props.put(ProducerConfig.ACKS_CONFIG, acks); props.put(ProducerConfig.RETRIES_CONFIG, retries); - props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); - props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs); - props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory); - if(other != null){ + + if (useConfluent) { + props.put("ssl.endpoint.identification.algorithm", sslAlgorithm); + props.put("sasl.mechanism", saslMechanism); + props.put("sasl.jaas.config", saslConfig); + props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, securityProtocol); + } else { + props.put(ProducerConfig.ACKS_CONFIG, acks); + props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); + props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs); + props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory); + } + + if (other != null) { other.forEach(kv -> props.put(kv.getKey(), kv.getValue())); } return props; diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index ac8d4fce78..a1fe74ae3f 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -39,6 +39,9 @@ function additionalComposeQueueArgs() { kafka) ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.kafka.yml" ;; + confluent) + ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.confluent.yml" + ;; aws-sqs) ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.aws-sqs.yml" ;; @@ -52,7 +55,7 @@ function additionalComposeQueueArgs() { ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml" ;; *) - echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or aws-sqs or pubsub or rabbitmq or service-bus." >&2 + echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or confluent or aws-sqs or pubsub or rabbitmq or service-bus." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_QUEUE_ARGS diff --git a/docker/docker-compose.confluent.yml b/docker/docker-compose.confluent.yml new file mode 100644 index 0000000000..1bbc3f96d6 --- /dev/null +++ b/docker/docker-compose.confluent.yml @@ -0,0 +1,57 @@ +# +# Copyright © 2016-2020 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-js-executor: + env_file: + - queue-confluent.env + tb-core1: + env_file: + - queue-confluent.env + depends_on: + - redis + tb-core2: + env_file: + - queue-confluent.env + depends_on: + - redis + tb-rule-engine1: + env_file: + - queue-confluent.env + depends_on: + - redis + tb-rule-engine2: + env_file: + - queue-confluent.env + depends_on: + - redis + tb-mqtt-transport1: + env_file: + - queue-confluent.env + tb-mqtt-transport2: + env_file: + - queue-confluent.env + tb-http-transport1: + env_file: + - queue-confluent.env + tb-http-transport2: + env_file: + - queue-confluent.env + tb-coap-transport: + env_file: + - queue-confluent.env diff --git a/docker/queue-confluent.env b/docker/queue-confluent.env new file mode 100644 index 0000000000..868a135de3 --- /dev/null +++ b/docker/queue-confluent.env @@ -0,0 +1,18 @@ +TB_QUEUE_TYPE=kafka + +TB_KAFKA_SERVERS=confluent.cloud:9092 +TB_QUEUE_KAFKA_REPLICATION_FACTOR=3 + +TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD=true +TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM=https +TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM=PLAIN +TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG=org.apache.kafka.common.security.plain.PlainLoginModule required username="CLUSTER_API_KEY" password="CLUSTER_API_SECRET"; +TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL=SASL_SSL +TB_QUEUE_KAFKA_CONFLUENT_USERNAME=CLUSTER_API_KEY +TB_QUEUE_KAFKA_CONFLUENT_PASSWORD=CLUSTER_API_SECRET + +TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000 +TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000 +TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000 +TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:1048576000 +TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600 diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index c573274801..ccd33177fd 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -26,6 +26,12 @@ kafka: servers: "TB_KAFKA_SERVERS" replication_factor: "TB_QUEUE_KAFKA_REPLICATION_FACTOR" topic_properties: "TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES" + use_confluent_cloud: "TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD" + confluent: + sasl: + mechanism: "TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM" + username: "TB_QUEUE_KAFKA_CONFLUENT_USERNAME" + password: "TB_QUEUE_KAFKA_CONFLUENT_PASSWORD" pubsub: project_id: "TB_QUEUE_PUBSUB_PROJECT_ID" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index f42b74745f..fac1ac7e8a 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -26,6 +26,10 @@ kafka: servers: "localhost:9092" replication_factor: "1" topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + use_confluent_cloud: false + confluent: + sasl: + mechanism: "PLAIN" pubsub: queue_properties: "ackDeadlineInSec:30;messageRetentionInSec:604800" diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 33637cee81..0420173188 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -61,15 +61,27 @@ function KafkaProducer() { const kafkaBootstrapServers = config.get('kafka.bootstrap.servers'); const requestTopic = config.get('request_topic'); + const useConfluent = config.get('kafka.use_confluent_cloud'); logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); logger.info('Kafka Requests Topic: %s', requestTopic); - kafkaClient = new Kafka({ + let kafkaConfig = { brokers: kafkaBootstrapServers.split(','), - logLevel: logLevel.INFO, - logCreator: KafkaJsWinstonLogCreator - }); + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator + }; + + if (useConfluent) { + kafkaConfig['sasl'] = { + mechanism: config.get('kafka.confluent.sasl.mechanism'), + username: config.get('kafka.confluent.username'), + password: config.get('kafka.confluent.password') + }; + kafkaConfig['ssl'] = true; + } + + kafkaClient = new Kafka(kafkaConfig); parseTopicProperties(); diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index c34dba7c90..f04a7559b8 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -69,6 +69,13 @@ queue: linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + other: topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" @@ -76,6 +83,7 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 377c66e711..a97a58cd85 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -62,6 +62,13 @@ queue: linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + other: topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" @@ -69,6 +76,7 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index bcc27d0755..b3804e3531 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -90,6 +90,13 @@ queue: linger.ms: "${TB_KAFKA_LINGER_MS:1}" buffer.memory: "${TB_BUFFER_MEMORY:33554432}" replication_factor: "${TB_QUEUE_KAFKA_REPLICATION_FACTOR:1}" + use_confluent_cloud: "${TB_QUEUE_KAFKA_USE_CONFLUENT_CLOUD:false}" + confluent: + ssl.algorithm: "${TB_QUEUE_KAFKA_CONFLUENT_SSL_ALGORITHM:https}" + sasl.mechanism: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_MECHANISM:PLAIN}" + sasl.config: "${TB_QUEUE_KAFKA_CONFLUENT_SASL_JAAS_CONFIG:org.apache.kafka.common.security.plain.PlainLoginModule required username=\"CLUSTER_API_KEY\" password=\"CLUSTER_API_SECRET\";}" + security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" + other: topic-properties: rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" @@ -97,6 +104,7 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" aws_sqs: + use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" secret_access_key: "${TB_QUEUE_AWS_SQS_SECRET_ACCESS_KEY:YOUR_SECRET}" region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" From 86779276cc4f047a8cadd11b31b0e1233cc87131 Mon Sep 17 00:00:00 2001 From: Valerii Sosliuk Date: Sat, 30 May 2020 15:13:24 +0300 Subject: [PATCH 23/39] Use metadata keys in originator attribute/telemetry nodes --- .../rule/engine/api/util/TbNodeUtils.java | 11 +++++++++++ .../metadata/TbAbstractGetAttributesNode.java | 9 +++++---- .../rule/engine/metadata/TbGetTelemetryNode.java | 13 +++++++------ 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/util/TbNodeUtils.java b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/util/TbNodeUtils.java index 871112c1b5..4e28afd93e 100644 --- a/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/util/TbNodeUtils.java +++ b/rule-engine/rule-engine-api/src/main/java/org/thingsboard/rule/engine/api/util/TbNodeUtils.java @@ -17,11 +17,15 @@ package org.thingsboard.rule.engine.api.util; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.util.CollectionUtils; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.server.common.msg.TbMsgMetaData; +import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * Created by ashvayka on 19.01.18. @@ -41,6 +45,13 @@ public class TbNodeUtils { } } + public static List processPatterns(List patterns, TbMsgMetaData metaData) { + if (!CollectionUtils.isEmpty(patterns)) { + return patterns.stream().map(p -> processPattern(p, metaData)).collect(Collectors.toList()); + } + return Collections.emptyList(); + } + public static String processPattern(String pattern, TbMsgMetaData metaData) { String result = new String(pattern); for (Map.Entry keyVal : metaData.values().entrySet()) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index 9c46cbbc1d..919c516ef3 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -29,6 +29,7 @@ import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNode; import org.thingsboard.rule.engine.api.TbNodeConfiguration; import org.thingsboard.rule.engine.api.TbNodeException; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; import org.thingsboard.server.common.data.kv.KvEntry; @@ -91,10 +92,10 @@ public abstract class TbAbstractGetAttributesNode> failuresMap = new ConcurrentHashMap<>(); ListenableFuture> allFutures = Futures.allAsList( - putLatestTelemetry(ctx, entityId, msg, LATEST_TS, config.getLatestTsKeyNames(), failuresMap), - putAttrAsync(ctx, entityId, msg, CLIENT_SCOPE, config.getClientAttributeNames(), failuresMap, "cs_"), - putAttrAsync(ctx, entityId, msg, SHARED_SCOPE, config.getSharedAttributeNames(), failuresMap, "shared_"), - putAttrAsync(ctx, entityId, msg, SERVER_SCOPE, config.getServerAttributeNames(), failuresMap, "ss_") + putLatestTelemetry(ctx, entityId, msg, LATEST_TS, TbNodeUtils.processPatterns(config.getLatestTsKeyNames(), msg.getMetaData()), failuresMap), + putAttrAsync(ctx, entityId, msg, CLIENT_SCOPE, TbNodeUtils.processPatterns(config.getClientAttributeNames(), msg.getMetaData()), failuresMap, "cs_"), + putAttrAsync(ctx, entityId, msg, SHARED_SCOPE, TbNodeUtils.processPatterns(config.getSharedAttributeNames(), msg.getMetaData()), failuresMap, "shared_"), + putAttrAsync(ctx, entityId, msg, SERVER_SCOPE, TbNodeUtils.processPatterns(config.getServerAttributeNames(), msg.getMetaData()), failuresMap, "ss_") ); withCallback(allFutures, i -> { if (!failuresMap.isEmpty()) { diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java index a3d84b25db..4d95e8a3d1 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbGetTelemetryNode.java @@ -103,9 +103,10 @@ public class TbGetTelemetryNode implements TbNode { if (config.isUseMetadataIntervalPatterns()) { checkMetadataKeyPatterns(msg); } - ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg)); + List keys = TbNodeUtils.processPatterns(tsKeyNames, msg.getMetaData()); + ListenableFuture> list = ctx.getTimeseriesService().findAll(ctx.getTenantId(), msg.getOriginator(), buildQueries(msg, keys)); DonAsynchron.withCallback(list, data -> { - process(data, msg); + process(data, msg, keys); ctx.tellSuccess(ctx.transformMsg(msg, msg.getType(), msg.getOriginator(), msg.getMetaData(), msg.getData())); }, error -> ctx.tellFailure(msg, error), ctx.getDbCallbackExecutor()); } catch (Exception e) { @@ -118,8 +119,8 @@ public class TbGetTelemetryNode implements TbNode { public void destroy() { } - private List buildQueries(TbMsg msg) { - return tsKeyNames.stream() + private List buildQueries(TbMsg msg, List keys) { + return keys.stream() .map(key -> new BaseReadTsKvQuery(key, getInterval(msg).getStartTs(), getInterval(msg).getEndTs(), 1, limit, NONE, getOrderBy())) .collect(Collectors.toList()); } @@ -135,7 +136,7 @@ public class TbGetTelemetryNode implements TbNode { } } - private void process(List entries, TbMsg msg) { + private void process(List entries, TbMsg msg, List keys) { ObjectNode resultNode = mapper.createObjectNode(); if (FETCH_MODE_ALL.equals(fetchMode)) { entries.forEach(entry -> processArray(resultNode, entry)); @@ -143,7 +144,7 @@ public class TbGetTelemetryNode implements TbNode { entries.forEach(entry -> processSingle(resultNode, entry)); } - for (String key : tsKeyNames) { + for (String key : keys) { if (resultNode.has(key)) { msg.getMetaData().putValue(key, resultNode.get(key).toString()); } From 197db609ca8656847ad1677310a9f885de66fa1a Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Tue, 11 Aug 2020 19:22:31 +0300 Subject: [PATCH 24/39] removed active field on Connect event reporting --- .../service/state/DefaultDeviceStateService.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 0025f80d71..1563d5abf4 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.state; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; @@ -503,8 +504,15 @@ public class DefaultDeviceStateService implements DeviceStateService { private void pushRuleEngineMessage(DeviceStateData stateData, String msgType) { DeviceState state = stateData.getState(); try { - TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON - , json.writeValueAsString(state)); + String data; + if (msgType.equals(CONNECT_EVENT)) { + ObjectNode stateNode = json.convertValue(state, ObjectNode.class); + stateNode.remove(ACTIVITY_STATE); + data = stateNode.toString(); + } else { + data = json.writeValueAsString(state); + } + TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON, data); clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); } catch (Exception e) { log.warn("[{}] Failed to push inactivity alarm: {}", stateData.getDeviceId(), state, e); From f4efea9a0bc62fecf7301d08deda40867ef67f75 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Wed, 12 Aug 2020 13:23:22 +0300 Subject: [PATCH 25/39] Improve connect event transformation --- .../state/DefaultDeviceStateService.java | 17 ++++++++--------- .../server/dao/util/mapping/JacksonUtil.java | 11 ++++++++++- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java index 1563d5abf4..9878bbf1a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/state/DefaultDeviceStateService.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.service.state; -import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Function; import com.google.common.util.concurrent.FutureCallback; @@ -46,16 +45,17 @@ import org.thingsboard.server.common.data.page.TextPageLink; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; +import org.thingsboard.server.common.msg.queue.ServiceType; +import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.dao.timeseries.TimeseriesService; +import org.thingsboard.server.dao.util.mapping.JacksonUtil; +import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.discovery.PartitionChangeEvent; import org.thingsboard.server.queue.discovery.PartitionService; -import org.thingsboard.server.common.msg.queue.ServiceType; -import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; -import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.queue.TbClusterService; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -91,7 +91,6 @@ import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; @Slf4j public class DefaultDeviceStateService implements DeviceStateService { - private static final ObjectMapper json = new ObjectMapper(); public static final String ACTIVITY_STATE = "active"; public static final String LAST_CONNECT_TIME = "lastConnectTime"; public static final String LAST_DISCONNECT_TIME = "lastDisconnectTime"; @@ -506,11 +505,11 @@ public class DefaultDeviceStateService implements DeviceStateService { try { String data; if (msgType.equals(CONNECT_EVENT)) { - ObjectNode stateNode = json.convertValue(state, ObjectNode.class); + ObjectNode stateNode = JacksonUtil.convertValue(state, ObjectNode.class); stateNode.remove(ACTIVITY_STATE); - data = stateNode.toString(); + data = JacksonUtil.toString(stateNode); } else { - data = json.writeValueAsString(state); + data = JacksonUtil.toString(state); } TbMsg tbMsg = TbMsg.newMsg(msgType, stateData.getDeviceId(), stateData.getMetaData().copy(), TbMsgDataType.JSON, data); clusterService.pushMsgToRuleEngine(stateData.getTenantId(), stateData.getDeviceId(), tbMsg, null); diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java index 7e4dfcdcf1..d654e85d5c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/mapping/JacksonUtil.java @@ -28,6 +28,15 @@ public class JacksonUtil { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static T convertValue(Object fromValue, Class toValueType) { + try { + return OBJECT_MAPPER.convertValue(fromValue, toValueType); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("The given object value: " + + fromValue + " cannot be converted to " + toValueType); + } + } + public static T fromString(String string, Class clazz) { try { return OBJECT_MAPPER.readValue(string, clazz); @@ -60,4 +69,4 @@ public class JacksonUtil { public static T clone(T value) { return fromString(toString(value), (Class) value.getClass()); } -} \ No newline at end of file +} From c6f037073b79388d73deef430e2963381f417999 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Fri, 29 May 2020 15:39:52 +0300 Subject: [PATCH 26/39] MQTT Gateway API tests --- .../server/msa/AbstractContainerTest.java | 39 ++ .../connectivity/MqttGatewayClientTest.java | 382 ++++++++++++++++++ 2 files changed, 421 insertions(+) create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index 6032dc112a..0184ac6d2a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -15,11 +15,15 @@ */ package org.thingsboard.server.msa; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; +import org.apache.cassandra.cql3.Json; import org.apache.commons.lang3.RandomStringUtils; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; @@ -32,6 +36,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; +import org.json.simple.JSONObject; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestRule; @@ -42,6 +47,7 @@ import org.thingsboard.rest.client.RestClient; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.msa.mapper.WsTelemetryResponse; @@ -52,6 +58,7 @@ import java.net.URI; import java.security.cert.X509Certificate; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Random; @Slf4j @@ -95,6 +102,17 @@ public abstract class AbstractContainerTest { } }; + protected Device createGatewayDevice() throws JsonProcessingException { + String isGateway = "{\"gateway\":true}"; + ObjectMapper objectMapper = new ObjectMapper(); + JsonNode additionalInfo = objectMapper.readTree(isGateway); + Device gatewayDeviceTemplate = new Device(); + gatewayDeviceTemplate.setName("mqtt_gateway"); + gatewayDeviceTemplate.setType("gateway"); + gatewayDeviceTemplate.setAdditionalInfo(additionalInfo); + return restClient.saveDevice(gatewayDeviceTemplate); + } + protected Device createDevice(String name) { return restClient.createDevice(name + RandomStringUtils.randomAlphanumeric(7), "DEFAULT"); } @@ -140,6 +158,27 @@ public abstract class AbstractContainerTest { return expectedValue.equals(list.get(1)); } + protected JsonObject createGatewayConnectPayload(String deviceName){ + JsonObject payload = new JsonObject(); + payload.addProperty("device", deviceName); + return payload; + } + + protected JsonObject createGatewayPayload(String deviceName, long ts){ + JsonObject payload = new JsonObject(); + payload.add(deviceName, createGatewayTelemetryArray(ts)); + return payload; + } + + protected JsonArray createGatewayTelemetryArray(long ts){ + JsonArray telemetryArray = new JsonArray(); + if (ts > 0) + telemetryArray.add(createPayload(ts)); + else + telemetryArray.add(createPayload()); + return telemetryArray; + } + protected JsonObject createPayload(long ts) { JsonObject values = createPayload(); JsonObject payload = new JsonObject(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java new file mode 100644 index 0000000000..d39a09f050 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -0,0 +1,382 @@ +/** + * Copyright © 2016-2020 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.connectivity; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Sets; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.mqtt.MqttQoS; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.checkerframework.checker.units.qual.A; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpMethod; +import org.springframework.http.ResponseEntity; +import org.thingsboard.mqtt.MqttClient; +import org.thingsboard.mqtt.MqttClientConfig; +import org.thingsboard.mqtt.MqttHandler; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.page.TextPageData; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.EntityRelationInfo; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.NodeConnectionInfo; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.rule.RuleNode; +import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.WsClient; +import org.thingsboard.server.msa.mapper.AttributesResponse; +import org.thingsboard.server.msa.mapper.WsTelemetryResponse; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.*; +import java.util.concurrent.*; + +@Slf4j +public class MqttGatewayClientTest extends AbstractContainerTest { + Device gatewayDevice; + MqttClient mqttClient; + Device createdDevice; + MqttMessageListener listener; + + @Before + public void createGateway() throws Exception { + restClient.login("tenant@thingsboard.org", "tenant"); + this.gatewayDevice = createGatewayDevice(); + Optional gatewayDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(gatewayDevice.getId()); + Assert.assertTrue(gatewayDeviceCredentials.isPresent()); + this.listener = new MqttMessageListener(); + this.mqttClient = getMqttClient(gatewayDeviceCredentials.get(), listener); + this.createdDevice = createDeviceThroughGateway(mqttClient, gatewayDevice); + } + + @After + public void removeGateway() throws Exception { + restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.gatewayDevice.getId()); + restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.createdDevice.getId()); + this.listener = null; + this.mqttClient = null; + this.createdDevice = null; + } + + @Test + public void telemetryUpload() throws Exception { + WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); + mqttClient.publish("v1/gateway/telemetry", Unpooled.wrappedBuffer(createGatewayPayload(createdDevice.getName(), -1).toString().getBytes())).get(); + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); + log.info("Received telemetry: {}", actualLatestTelemetry); + wsClient.closeBlocking(); + + Assert.assertEquals(4, actualLatestTelemetry.getData().size()); + Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"), + actualLatestTelemetry.getLatestValues().keySet()); + + Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString())); + Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1")); + Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0))); + Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73))); + } + + @Test + public void telemetryUploadWithTs() throws Exception { + long ts = 1451649600512L; + + restClient.login("tenant@thingsboard.org", "tenant"); + WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); + mqttClient.publish("v1/gateway/telemetry", Unpooled.wrappedBuffer(createGatewayPayload(createdDevice.getName(), ts).toString().getBytes())).get(); + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); + log.info("Received telemetry: {}", actualLatestTelemetry); + wsClient.closeBlocking(); + + Assert.assertEquals(4, actualLatestTelemetry.getData().size()); + Assert.assertEquals(getExpectedLatestValues(ts), actualLatestTelemetry.getLatestValues()); + + Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", ts, Boolean.TRUE.toString())); + Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", ts, "value1")); + Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", ts, Double.toString(42.0))); + Assert.assertTrue(verify(actualLatestTelemetry, "longKey", ts, Long.toString(73))); + } + + @Test + public void publishAttributeUpdateToServer() throws Exception { + Optional createdDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); + Assert.assertTrue(createdDeviceCredentials.isPresent()); + WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); + JsonObject clientAttributes = new JsonObject(); + clientAttributes.addProperty("attr1", "value1"); + clientAttributes.addProperty("attr2", true); + clientAttributes.addProperty("attr3", 42.0); + clientAttributes.addProperty("attr4", 73); + JsonObject gatewayClientAttributes = new JsonObject(); + gatewayClientAttributes.add(createdDevice.getName(), clientAttributes); + mqttClient.publish("v1/gateway/attributes", Unpooled.wrappedBuffer(gatewayClientAttributes.toString().getBytes())).get(); + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); + log.info("Received attributes: {}", actualLatestTelemetry); + wsClient.closeBlocking(); + + Assert.assertEquals(4, actualLatestTelemetry.getData().size()); + Assert.assertEquals(Sets.newHashSet("attr1", "attr2", "attr3", "attr4"), + actualLatestTelemetry.getLatestValues().keySet()); + + Assert.assertTrue(verify(actualLatestTelemetry, "attr1", "value1")); + Assert.assertTrue(verify(actualLatestTelemetry, "attr2", Boolean.TRUE.toString())); + Assert.assertTrue(verify(actualLatestTelemetry, "attr3", Double.toString(42.0))); + Assert.assertTrue(verify(actualLatestTelemetry, "attr4", Long.toString(73))); + } + + @Test + public void requestAttributeValuesFromServer() throws Exception { + WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); + // Add a new client attribute + JsonObject clientAttributes = new JsonObject(); + String clientAttributeValue = RandomStringUtils.randomAlphanumeric(8); + clientAttributes.addProperty("clientAttr", clientAttributeValue); + + JsonObject gatewayClientAttributes = new JsonObject(); + gatewayClientAttributes.add(createdDevice.getName(), clientAttributes); + mqttClient.publish("v1/gateway/attributes", Unpooled.wrappedBuffer(gatewayClientAttributes.toString().getBytes())).get(); + + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); + log.info("Received ws telemetry: {}", actualLatestTelemetry); + wsClient.closeBlocking(); + + Assert.assertEquals(1, actualLatestTelemetry.getData().size()); + Assert.assertEquals(Sets.newHashSet("clientAttr"), + actualLatestTelemetry.getLatestValues().keySet()); + + Assert.assertTrue(verify(actualLatestTelemetry, "clientAttr", clientAttributeValue)); + + // Add a new shared attribute + JsonObject sharedAttributes = new JsonObject(); + String sharedAttributeValue = RandomStringUtils.randomAlphanumeric(8); + sharedAttributes.addProperty("sharedAttr", sharedAttributeValue); + + ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() + .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", + mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, + createdDevice.getId()); + Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + MqttEvent sharedAttributeEvent = listener.getEvents().poll(10, TimeUnit.SECONDS); + + // Catch attribute update event + Assert.assertNotNull(sharedAttributeEvent); + Assert.assertEquals("v1/gateway/attributes", sharedAttributeEvent.getTopic()); + + // Subscribe to attributes response + mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); + + // Wait until subscription is processed + TimeUnit.SECONDS.sleep(3); + + checkAttribute(true, clientAttributeValue); + checkAttribute(false, sharedAttributeValue); + } + + @Test + public void subscribeToAttributeUpdatesFromServer() throws Exception { + mqttClient.on("v1/gateway/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(); + // Wait until subscription is processed + TimeUnit.SECONDS.sleep(3); + String sharedAttributeName = "sharedAttr"; + // Add a new shared attribute + + JsonObject sharedAttributes = new JsonObject(); + String sharedAttributeValue = RandomStringUtils.randomAlphanumeric(8); + sharedAttributes.addProperty(sharedAttributeName, sharedAttributeValue); + + JsonObject gatewaySharedAttributeValue = new JsonObject(); + gatewaySharedAttributeValue.addProperty("device", createdDevice.getName()); + gatewaySharedAttributeValue.add("data", sharedAttributes); + + ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() + .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", + mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, + createdDevice.getId()); + Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + + MqttEvent event = listener.getEvents().poll(10, TimeUnit.SECONDS); + Assert.assertEquals(sharedAttributeValue, + mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()); + + // Update the shared attribute value + JsonObject updatedSharedAttributes = new JsonObject(); + String updatedSharedAttributeValue = RandomStringUtils.randomAlphanumeric(8); + updatedSharedAttributes.addProperty(sharedAttributeName, updatedSharedAttributeValue); + + JsonObject gatewayUpdatedSharedAttributeValue = new JsonObject(); + gatewayUpdatedSharedAttributeValue.addProperty("device", createdDevice.getName()); + gatewayUpdatedSharedAttributeValue.add("data", updatedSharedAttributes); + + ResponseEntity updatedSharedAttributesResponse = restClient.getRestTemplate() + .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", + mapper.readTree(updatedSharedAttributes.toString()), ResponseEntity.class, + createdDevice.getId()); + Assert.assertTrue(updatedSharedAttributesResponse.getStatusCode().is2xxSuccessful()); + + event = listener.getEvents().poll(10, TimeUnit.SECONDS); + Assert.assertEquals(updatedSharedAttributeValue, + mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()); + } + + @Test + public void serverSideRpc() throws Exception { + String gatewayRpcTopic = "v1/gateway/rpc"; + mqttClient.on(gatewayRpcTopic, listener, MqttQoS.AT_LEAST_ONCE).get(); + + // Wait until subscription is processed + TimeUnit.SECONDS.sleep(3); + + // Send an RPC from the server + JsonObject serverRpcPayload = new JsonObject(); + serverRpcPayload.addProperty("method", "getValue"); + serverRpcPayload.addProperty("params", true); + ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor()); + ListenableFuture future = service.submit(() -> { + try { + return restClient.getRestTemplate() + .postForEntity(HTTPS_URL + "/api/plugins/rpc/twoway/{deviceId}", + mapper.readTree(serverRpcPayload.toString()), String.class, + createdDevice.getId()); + } catch (IOException e) { + return ResponseEntity.badRequest().build(); + } + }); + + // Wait for RPC call from the server and send the response + MqttEvent requestFromServer = listener.getEvents().poll(10, TimeUnit.SECONDS); + + Assert.assertNotNull(requestFromServer); + Assert.assertNotNull(requestFromServer.getMessage()); + + JsonObject requestFromServerJson = new JsonParser().parse(requestFromServer.getMessage()).getAsJsonObject(); + + Assert.assertEquals(createdDevice.getName(), requestFromServerJson.get("device").getAsString()); + + JsonObject requestFromServerData = requestFromServerJson.get("data").getAsJsonObject(); + + Assert.assertEquals("getValue", requestFromServerData.get("method").getAsString()); + Assert.assertTrue(requestFromServerData.get("params").getAsBoolean()); + + int requestId = requestFromServerData.get("id").getAsInt(); + + JsonObject clientResponse = new JsonObject(); + clientResponse.addProperty("response", "someResponse"); + JsonObject gatewayResponse = new JsonObject(); + gatewayResponse.addProperty("device", createdDevice.getName()); + gatewayResponse.addProperty("id", requestId); + gatewayResponse.add("data", clientResponse); + // Send a response to the server's RPC request + + mqttClient.publish(gatewayRpcTopic, Unpooled.wrappedBuffer(gatewayResponse.toString().getBytes())).get(); + ResponseEntity serverResponse = future.get(5, TimeUnit.SECONDS); + Assert.assertTrue(serverResponse.getStatusCode().is2xxSuccessful()); + Assert.assertEquals(clientResponse.toString(), serverResponse.getBody()); + } + + private void checkAttribute(boolean client, String expectedValue) throws Exception{ + JsonObject gatewayAttributesRequest = new JsonObject(); + int messageId = new Random().nextInt(100); + gatewayAttributesRequest.addProperty("id", messageId); + gatewayAttributesRequest.addProperty("device", createdDevice.getName()); + gatewayAttributesRequest.addProperty("client", client); + String attributeName; + if (client) + attributeName = "clientAttr"; + else + attributeName = "sharedAttr"; + gatewayAttributesRequest.addProperty("key", attributeName); + log.info(gatewayAttributesRequest.toString()); + mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get(); + MqttEvent clientAttributeEvent = listener.getEvents().poll(10, TimeUnit.SECONDS); + Assert.assertNotNull(clientAttributeEvent); + JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject(); + + Assert.assertEquals(messageId, responseMessage.get("id").getAsInt()); + Assert.assertEquals(createdDevice.getName(), responseMessage.get("device").getAsString()); + Assert.assertEquals(3, responseMessage.entrySet().size()); + Assert.assertEquals(expectedValue, responseMessage.get("value").getAsString()); + } + + private Device createDeviceThroughGateway(MqttClient mqttClient, Device gatewayDevice) throws Exception { + String deviceName = "mqtt_device"; + mqttClient.publish("v1/gateway/connect", Unpooled.wrappedBuffer(createGatewayConnectPayload(deviceName).toString().getBytes())).get(); + + TimeUnit.SECONDS.sleep(3); + List relations = restClient.findByFrom(gatewayDevice.getId(), RelationTypeGroup.COMMON); + + Assert.assertEquals(1, relations.size()); + + EntityId createdEntityId = relations.get(0).getTo(); + DeviceId createdDeviceId = new DeviceId(createdEntityId.getId()); + Optional createdDevice = restClient.getDeviceById(createdDeviceId); + + Assert.assertTrue(createdDevice.isPresent()); + + return createdDevice.get(); + } + + private MqttClient getMqttClient(DeviceCredentials deviceCredentials, MqttMessageListener listener) throws InterruptedException, ExecutionException { + MqttClientConfig clientConfig = new MqttClientConfig(); + clientConfig.setClientId("MQTT client from test"); + clientConfig.setUsername(deviceCredentials.getCredentialsId()); + MqttClient mqttClient = MqttClient.create(clientConfig, listener); + mqttClient.connect("localhost", 1883).get(); + return mqttClient; + } + + @Data + private class MqttMessageListener implements MqttHandler { + private final BlockingQueue events; + + private MqttMessageListener() { + events = new ArrayBlockingQueue<>(100); + } + + @Override + public void onMessage(String topic, ByteBuf message) { + log.info("MQTT message [{}], topic [{}]", message.toString(StandardCharsets.UTF_8), topic); + events.add(new MqttEvent(topic, message.toString(StandardCharsets.UTF_8))); + } + + } + + @Data + private class MqttEvent { + private final String topic; + private final String message; + } + + +} From 92a7ee9e7dbcf8de0e4a72996d4ac03e77a8bd2f Mon Sep 17 00:00:00 2001 From: Valerii Sosliuk Date: Fri, 14 Aug 2020 20:30:49 +0300 Subject: [PATCH 27/39] UI: Attribute Service pass config on delete attributes --- ui/src/app/api/attribute.service.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/app/api/attribute.service.js b/ui/src/app/api/attribute.service.js index 54655d7e29..aa2286d938 100644 --- a/ui/src/app/api/attribute.service.js +++ b/ui/src/app/api/attribute.service.js @@ -228,7 +228,7 @@ function AttributeService($http, $q, $filter, types, telemetryWebsocketService) } var deleteEntityAttributesPromise; if (deleteAttributes.length) { - deleteEntityAttributesPromise = deleteEntityAttributes(entityType, entityId, attributeScope, deleteAttributes); + deleteEntityAttributesPromise = deleteEntityAttributes(entityType, entityId, attributeScope, deleteAttributes, config); } if (Object.keys(attributesData).length) { var url = '/api/plugins/telemetry/' + entityType + '/' + entityId + '/' + attributeScope; From 949790db1ff57f7d6dcfd173e680d901f4875f90 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 17 Aug 2020 14:01:37 +0300 Subject: [PATCH 28/39] not send smtp password to UI --- .../thingsboard/server/controller/AdminController.java | 8 +++++++- .../server/dao/settings/AdminSettingsServiceImpl.java | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 8873d82366..134171c8b1 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.controller; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; @@ -59,7 +60,11 @@ public class AdminController extends BaseController { public AdminSettings getAdminSettings(@PathVariable("key") String key) throws ThingsboardException { try { accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); - return checkNotNull(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, key)); + AdminSettings adminSettings = checkNotNull(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, key)); + if (adminSettings.getKey().equals("mail")) { + ((ObjectNode) adminSettings.getJsonValue()).put("password", ""); + } + return adminSettings; } catch (Exception e) { throw handleException(e); } @@ -74,6 +79,7 @@ public class AdminController extends BaseController { adminSettings = checkNotNull(adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings)); if (adminSettings.getKey().equals("mail")) { mailService.updateMailConfiguration(); + ((ObjectNode) adminSettings.getJsonValue()).put("password", ""); } return adminSettings; } catch (Exception e) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 6c7495cc19..4374e314cf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.settings; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -52,6 +53,13 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { public AdminSettings saveAdminSettings(TenantId tenantId, AdminSettings adminSettings) { log.trace("Executing saveAdminSettings [{}]", adminSettings); adminSettingsValidator.validate(adminSettings, data -> tenantId); + if (adminSettings.getKey().equals("mail") && "".equals(adminSettings.getJsonValue().get("password").asText())) { + AdminSettings mailSettings = findAdminSettingsByKey(tenantId, "mail"); + if (mailSettings != null) { + ((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText()); + } + } + return adminSettingsDao.save(tenantId, adminSettings); } From d8f2da4864c0349513df4461fd6d4935d9d09014 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 17 Aug 2020 17:28:29 +0300 Subject: [PATCH 29/39] refactored UI for mail settings --- ui/src/app/admin/outgoing-mail-settings.tpl.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui/src/app/admin/outgoing-mail-settings.tpl.html b/ui/src/app/admin/outgoing-mail-settings.tpl.html index 596c9c4091..5f8668bc3d 100644 --- a/ui/src/app/admin/outgoing-mail-settings.tpl.html +++ b/ui/src/app/admin/outgoing-mail-settings.tpl.html @@ -123,16 +123,16 @@ - + - + - +
{{'admin.send-test-mail' | translate}} From b341030c8feb1013b429947d72b747e02196ca60 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 20 Aug 2020 16:52:41 +0300 Subject: [PATCH 30/39] Version set to 2.5.4-SNAPSHOT --- application/pom.xml | 2 +- common/actor/pom.xml | 2 +- common/dao-api/pom.xml | 2 +- common/data/pom.xml | 2 +- common/message/pom.xml | 2 +- common/pom.xml | 2 +- common/queue/pom.xml | 2 +- common/stats/pom.xml | 2 +- common/transport/coap/pom.xml | 2 +- common/transport/http/pom.xml | 2 +- common/transport/mqtt/pom.xml | 2 +- common/transport/pom.xml | 2 +- common/transport/transport-api/pom.xml | 2 +- common/util/pom.xml | 2 +- dao/pom.xml | 2 +- msa/black-box-tests/pom.xml | 2 +- msa/js-executor/package.json | 2 +- msa/js-executor/pom.xml | 2 +- msa/pom.xml | 2 +- msa/tb-node/pom.xml | 2 +- msa/tb/pom.xml | 2 +- msa/transport/coap/pom.xml | 2 +- msa/transport/http/pom.xml | 2 +- msa/transport/mqtt/pom.xml | 2 +- msa/transport/pom.xml | 2 +- msa/web-ui/package.json | 2 +- msa/web-ui/pom.xml | 2 +- netty-mqtt/pom.xml | 4 ++-- pom.xml | 2 +- rest-client/pom.xml | 2 +- rule-engine/pom.xml | 2 +- rule-engine/rule-engine-api/pom.xml | 2 +- rule-engine/rule-engine-components/pom.xml | 2 +- tools/pom.xml | 2 +- transport/coap/pom.xml | 2 +- transport/http/pom.xml | 2 +- transport/mqtt/pom.xml | 2 +- transport/pom.xml | 2 +- ui/package.json | 2 +- ui/pom.xml | 2 +- 40 files changed, 41 insertions(+), 41 deletions(-) diff --git a/application/pom.xml b/application/pom.xml index 311a3d4e06..ffb60b169a 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard application diff --git a/common/actor/pom.xml b/common/actor/pom.xml index 9d101a2edb..542715c2f4 100644 --- a/common/actor/pom.xml +++ b/common/actor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/common/dao-api/pom.xml b/common/dao-api/pom.xml index b880c42bfd..204438477f 100644 --- a/common/dao-api/pom.xml +++ b/common/dao-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/common/data/pom.xml b/common/data/pom.xml index 96892bdc59..acd856e14e 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/common/message/pom.xml b/common/message/pom.xml index 066beec7a1..2df71c6332 100644 --- a/common/message/pom.xml +++ b/common/message/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/common/pom.xml b/common/pom.xml index de6542da2b..67f43bf6a6 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard common diff --git a/common/queue/pom.xml b/common/queue/pom.xml index 3d9a1a97ff..f2ebc7ae54 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/common/stats/pom.xml b/common/stats/pom.xml index 1e8fc57115..7a38c27c2a 100644 --- a/common/stats/pom.xml +++ b/common/stats/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/coap/pom.xml b/common/transport/coap/pom.xml index 5ec305329c..1e1753a0cc 100644 --- a/common/transport/coap/pom.xml +++ b/common/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/http/pom.xml b/common/transport/http/pom.xml index 4089f3c0d8..8a4bb46cd9 100644 --- a/common/transport/http/pom.xml +++ b/common/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/mqtt/pom.xml b/common/transport/mqtt/pom.xml index 7a3e81348c..ccf1f7f804 100644 --- a/common/transport/mqtt/pom.xml +++ b/common/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/pom.xml b/common/transport/pom.xml index 3dfb3ee695..b5e89fe709 100644 --- a/common/transport/pom.xml +++ b/common/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 3681de25e6..b1d8c6b7bc 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/util/pom.xml b/common/util/pom.xml index 25b8925b9a..317c9a061e 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT common org.thingsboard.common diff --git a/dao/pom.xml b/dao/pom.xml index c8b46526c6..479eecf94f 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard dao diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 4880410053..1e2b5e26b3 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index cedef9f059..599968a367 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-js-executor", "private": true, - "version": "2.5.3", + "version": "2.5.4", "description": "ThingsBoard JavaScript Executor Microservice", "main": "server.js", "bin": "server.js", diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index ab810ab4da..12a1e33de5 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/pom.xml b/msa/pom.xml index deee02a8a4..4221d1c0ea 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard msa diff --git a/msa/tb-node/pom.xml b/msa/tb-node/pom.xml index 4a20d19a90..575beeeb08 100644 --- a/msa/tb-node/pom.xml +++ b/msa/tb-node/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/tb/pom.xml b/msa/tb/pom.xml index b27fe66615..da384db8df 100644 --- a/msa/tb/pom.xml +++ b/msa/tb/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/transport/coap/pom.xml b/msa/transport/coap/pom.xml index 373e6416ff..3575a36f9b 100644 --- a/msa/transport/coap/pom.xml +++ b/msa/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/http/pom.xml b/msa/transport/http/pom.xml index e3e1d47a9b..bee5188f63 100644 --- a/msa/transport/http/pom.xml +++ b/msa/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/mqtt/pom.xml b/msa/transport/mqtt/pom.xml index b84d064f31..5eeb52b724 100644 --- a/msa/transport/mqtt/pom.xml +++ b/msa/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/pom.xml b/msa/transport/pom.xml index c4c17d5b3e..f45a68e2fd 100644 --- a/msa/transport/pom.xml +++ b/msa/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 749fb2df72..acbe02cafb 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-web-ui", "private": true, - "version": "2.5.3", + "version": "2.5.4", "description": "ThingsBoard Web UI Microservice", "main": "server.js", "bin": "server.js", diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index 54e3094db8..aeabf26ab9 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT msa org.thingsboard.msa diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index eb32d84f20..701d128380 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -19,11 +19,11 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard netty-mqtt - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT jar Netty MQTT Client diff --git a/pom.xml b/pom.xml index 209df5b52d..640609c78b 100755 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT pom Thingsboard diff --git a/rest-client/pom.xml b/rest-client/pom.xml index edb6e9be2b..bc60de59ed 100644 --- a/rest-client/pom.xml +++ b/rest-client/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard rest-client diff --git a/rule-engine/pom.xml b/rule-engine/pom.xml index 66f8cec913..d73c3dfe78 100644 --- a/rule-engine/pom.xml +++ b/rule-engine/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard rule-engine diff --git a/rule-engine/rule-engine-api/pom.xml b/rule-engine/rule-engine-api/pom.xml index 49386f1f86..1bd34eb2d3 100644 --- a/rule-engine/rule-engine-api/pom.xml +++ b/rule-engine/rule-engine-api/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 1b6fc2fc74..f38fe14d22 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/tools/pom.xml b/tools/pom.xml index 6cf3467015..533f802c2e 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard tools diff --git a/transport/coap/pom.xml b/transport/coap/pom.xml index f265d6e9cf..a6d69e20ed 100644 --- a/transport/coap/pom.xml +++ b/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/http/pom.xml b/transport/http/pom.xml index fddc86a8da..4869f01fe2 100644 --- a/transport/http/pom.xml +++ b/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/mqtt/pom.xml b/transport/mqtt/pom.xml index cb37b62a34..141d0ae6b0 100644 --- a/transport/mqtt/pom.xml +++ b/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/pom.xml b/transport/pom.xml index 9f2d726073..5b4cbec2a2 100644 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard transport diff --git a/ui/package.json b/ui/package.json index c507bbe7e9..e3691c5ef8 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard", "private": true, - "version": "2.5.3", + "version": "2.5.4", "description": "ThingsBoard UI", "licenses": [ { diff --git a/ui/pom.xml b/ui/pom.xml index 00b8d3e87b..3bf9eced54 100644 --- a/ui/pom.xml +++ b/ui/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.3-SNAPSHOT + 2.5.4-SNAPSHOT thingsboard org.thingsboard From e33709d830d25d8fa9b998e80f4898168db63b63 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Thu, 27 Aug 2020 17:11:00 +0300 Subject: [PATCH 31/39] fix/getAttributesRequest & fix/hsql-inserts-with-json (#3376) * fix GetAttributeRequest & Hsql inserts * added deleted keys to GetAttributeResponseMsg for gateway * changed logic for handleGetAttributesRequest on deleted attributes * removed unused classes and remove deleted keys in GetAttributeResponseMsg proto * revert yml file * removed getDeletedAttributeKeysCount method call in Coap transport --- .../device/DeviceActorMessageProcessor.java | 21 ++++---- .../transport/DefaultTransportApiService.java | 2 +- .../server/common/msg/kv/AttributesKVMsg.java | 29 ----------- .../common/msg/kv/BasicAttributeKVMsg.java | 52 ------------------- common/queue/src/main/proto/queue.proto | 1 - .../coap/adaptors/JsonCoapAdaptor.java | 2 +- .../transport/adaptor/JsonConverter.java | 35 ------------- .../dao/sqlts/hsql/JpaHsqlTimeseriesDao.java | 1 + .../hsql/HsqlLatestInsertTsRepository.java | 4 +- .../resources/sql/hsql/drop-all-tables.sql | 1 + 10 files changed, 16 insertions(+), 132 deletions(-) delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/kv/AttributesKVMsg.java delete mode 100644 common/message/src/main/java/org/thingsboard/server/common/msg/kv/BasicAttributeKVMsg.java diff --git a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java index e328931c0e..cf4aa3edda 100644 --- a/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java +++ b/application/src/main/java/org/thingsboard/server/actors/device/DeviceActorMessageProcessor.java @@ -28,6 +28,7 @@ import org.thingsboard.rule.engine.api.msg.DeviceNameOrTypeUpdateMsg; import org.thingsboard.server.actors.ActorSystemContext; import org.thingsboard.server.actors.TbActorCtx; import org.thingsboard.server.actors.shared.AbstractContextAwareMsgProcessor; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -79,8 +80,6 @@ import java.util.UUID; import java.util.function.Consumer; import java.util.stream.Collectors; -import static org.thingsboard.server.common.data.DataConstants.CLIENT_SCOPE; -import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; /** * @author Andrew Shvayka @@ -279,17 +278,17 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { ListenableFuture> clientAttributesFuture; ListenableFuture> sharedAttributesFuture; if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) { - clientAttributesFuture = findAllAttributesByScope(CLIENT_SCOPE); - sharedAttributesFuture = findAllAttributesByScope(SHARED_SCOPE); + clientAttributesFuture = findAllAttributesByScope(DataConstants.CLIENT_SCOPE); + sharedAttributesFuture = findAllAttributesByScope(DataConstants.SHARED_SCOPE); } else if (!CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) { - clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), CLIENT_SCOPE); - sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), SHARED_SCOPE); + clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), DataConstants.CLIENT_SCOPE); + sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), DataConstants.SHARED_SCOPE); } else if (CollectionUtils.isEmpty(request.getClientAttributeNamesList()) && !CollectionUtils.isEmpty(request.getSharedAttributeNamesList())) { clientAttributesFuture = Futures.immediateFuture(Collections.emptyList()); - sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), SHARED_SCOPE); + sharedAttributesFuture = findAttributesByScope(toSet(request.getSharedAttributeNamesList()), DataConstants.SHARED_SCOPE); } else { sharedAttributesFuture = Futures.immediateFuture(Collections.emptyList()); - clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), CLIENT_SCOPE); + clientAttributesFuture = findAttributesByScope(toSet(request.getClientAttributeNamesList()), DataConstants.CLIENT_SCOPE); } return Futures.allAsList(Arrays.asList(clientAttributesFuture, sharedAttributesFuture)); } @@ -316,7 +315,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { AttributeUpdateNotificationMsg.Builder notification = AttributeUpdateNotificationMsg.newBuilder(); if (msg.isDeleted()) { List sharedKeys = msg.getDeletedKeys().stream() - .filter(key -> SHARED_SCOPE.equals(key.getScope())) + .filter(key -> DataConstants.SHARED_SCOPE.equals(key.getScope())) .map(AttributeKey::getAttributeKey) .collect(Collectors.toList()); if (!sharedKeys.isEmpty()) { @@ -324,7 +323,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { hasNotificationData = true; } } else { - if (SHARED_SCOPE.equals(msg.getScope())) { + if (DataConstants.SHARED_SCOPE.equals(msg.getScope())) { List attributes = new ArrayList<>(msg.getValues()); if (attributes.size() > 0) { List sharedUpdated = msg.getValues().stream().map(this::toTsKvProto) @@ -334,7 +333,7 @@ class DeviceActorMessageProcessor extends AbstractContextAwareMsgProcessor { hasNotificationData = true; } } else { - log.debug("[{}] No public server side attributes changed!", deviceId); + log.debug("[{}] No public shared side attributes changed!", deviceId); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index b08ff7c20f..3aa14ed639 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -154,7 +154,7 @@ public class DefaultTransportApiService implements TransportApiService { return TransportApiResponseMsg.newBuilder() .setGetOrCreateDeviceResponseMsg(GetOrCreateDeviceFromGatewayResponseMsg.newBuilder().setDeviceInfo(getDeviceInfoProto(device)).build()).build(); } catch (JsonProcessingException e) { - log.warn("[{}] Failed to lookup device by gateway id and name", gatewayId, requestMsg.getDeviceName(), e); + log.warn("[{}][{}] Failed to lookup device by gateway id and name", gatewayId, requestMsg.getDeviceName(), e); throw new RuntimeException(e); } finally { deviceCreationLock.unlock(); diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/kv/AttributesKVMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/kv/AttributesKVMsg.java deleted file mode 100644 index 85283242f9..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/kv/AttributesKVMsg.java +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.kv; - -import java.io.Serializable; -import java.util.List; - -import org.thingsboard.server.common.data.kv.AttributeKey; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; - -public interface AttributesKVMsg extends Serializable { - - List getClientAttributes(); - List getSharedAttributes(); - List getDeletedAttributes(); -} diff --git a/common/message/src/main/java/org/thingsboard/server/common/msg/kv/BasicAttributeKVMsg.java b/common/message/src/main/java/org/thingsboard/server/common/msg/kv/BasicAttributeKVMsg.java deleted file mode 100644 index 8eb087f961..0000000000 --- a/common/message/src/main/java/org/thingsboard/server/common/msg/kv/BasicAttributeKVMsg.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Copyright © 2016-2020 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.common.msg.kv; - -import lombok.AccessLevel; -import lombok.Data; -import lombok.RequiredArgsConstructor; -import org.thingsboard.server.common.data.kv.AttributeKey; -import org.thingsboard.server.common.data.kv.AttributeKvEntry; - -import java.util.Collections; -import java.util.List; - -@Data -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public class BasicAttributeKVMsg implements AttributesKVMsg { - - private static final long serialVersionUID = 1L; - - private final List clientAttributes; - private final List sharedAttributes; - private final List deletedAttributes; - - public static BasicAttributeKVMsg fromClient(List attributes) { - return new BasicAttributeKVMsg(attributes, Collections.emptyList(), Collections.emptyList()); - } - - public static BasicAttributeKVMsg fromShared(List attributes) { - return new BasicAttributeKVMsg(Collections.emptyList(), attributes, Collections.emptyList()); - } - - public static BasicAttributeKVMsg from(List client, List shared) { - return new BasicAttributeKVMsg(client, shared, Collections.emptyList()); - } - - public static AttributesKVMsg fromDeleted(List shared) { - return new BasicAttributeKVMsg(Collections.emptyList(), Collections.emptyList(), shared); - } -} diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index d3e850b39c..59d45d7db6 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -127,7 +127,6 @@ message GetAttributeResponseMsg { int32 requestId = 1; repeated TsKvProto clientAttributeList = 2; repeated TsKvProto sharedAttributeList = 3; - repeated string deletedAttributeKeys = 4; string error = 5; } diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java index 9d509d8f60..5c9c471570 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/adaptors/JsonCoapAdaptor.java @@ -125,7 +125,7 @@ public class JsonCoapAdaptor implements CoapTransportAdaptor { @Override public Response convertToPublish(CoapTransportResource.CoapSessionListener session, TransportProtos.GetAttributeResponseMsg msg) throws AdaptorException { - if (msg.getClientAttributeListCount() == 0 && msg.getSharedAttributeListCount() == 0 && msg.getDeletedAttributeKeysCount() == 0) { + if (msg.getClientAttributeListCount() == 0 && msg.getSharedAttributeListCount() == 0) { return new Response(CoAP.ResponseCode.NOT_FOUND); } else { Response response = new Response(CoAP.ResponseCode.CONTENT); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 8375b84ffa..a43437bb9d 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -35,7 +35,6 @@ import org.thingsboard.server.common.data.kv.JsonDataEntry; import org.thingsboard.server.common.data.kv.KvEntry; import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; -import org.thingsboard.server.common.msg.kv.AttributesKVMsg; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; @@ -269,11 +268,6 @@ public class JsonConverter { payload.getSharedAttributeListList().forEach(addToObjectFromProto(attrObject)); result.add("shared", attrObject); } - if (payload.getDeletedAttributeKeysCount() > 0) { - JsonArray attrObject = new JsonArray(); - payload.getDeletedAttributeKeysList().forEach(attrObject::add); - result.add("deleted", attrObject); - } return result; } @@ -290,31 +284,6 @@ public class JsonConverter { return result; } - public static JsonObject toJson(AttributesKVMsg payload, boolean asMap) { - JsonObject result = new JsonObject(); - if (asMap) { - if (!payload.getClientAttributes().isEmpty()) { - JsonObject attrObject = new JsonObject(); - payload.getClientAttributes().forEach(addToObject(attrObject)); - result.add("client", attrObject); - } - if (!payload.getSharedAttributes().isEmpty()) { - JsonObject attrObject = new JsonObject(); - payload.getSharedAttributes().forEach(addToObject(attrObject)); - result.add("shared", attrObject); - } - } else { - payload.getClientAttributes().forEach(addToObject(result)); - payload.getSharedAttributes().forEach(addToObject(result)); - } - if (!payload.getDeletedAttributes().isEmpty()) { - JsonArray attrObject = new JsonArray(); - payload.getDeletedAttributes().forEach(addToObject(attrObject)); - result.add("deleted", attrObject); - } - return result; - } - public static JsonObject getJsonObjectForGateway(String deviceName, TransportProtos.GetAttributeResponseMsg responseMsg) { JsonObject result = new JsonObject(); result.addProperty("id", responseMsg.getRequestId()); @@ -370,10 +339,6 @@ public class JsonConverter { } } - private static Consumer addToObject(JsonArray result) { - return key -> result.add(key.getAttributeKey()); - } - private static Consumer addToObjectFromProto(JsonObject result) { return de -> { switch (de.getKv().getType()) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java index 5d94d12a1a..5ea387328d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/hsql/JpaHsqlTimeseriesDao.java @@ -46,6 +46,7 @@ public class JpaHsqlTimeseriesDao extends AbstractChunkedAggregationTimeseriesDa entity.setDoubleValue(tsKvEntry.getDoubleValue().orElse(null)); entity.setLongValue(tsKvEntry.getLongValue().orElse(null)); entity.setBooleanValue(tsKvEntry.getBooleanValue().orElse(null)); + entity.setJsonValue(tsKvEntry.getJsonValue().orElse(null)); log.trace("Saving entity: {}", entity); return tsQueue.add(entity); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java index 224dc52805..429d5de31b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/insert/latest/hsql/HsqlLatestInsertTsRepository.java @@ -41,8 +41,8 @@ public class HsqlLatestInsertTsRepository extends AbstractInsertRepository imple "ON (ts_kv_latest.entity_id=T.entity_id " + "AND ts_kv_latest.key=T.key) " + "WHEN MATCHED THEN UPDATE SET ts_kv_latest.ts = T.ts, ts_kv_latest.bool_v = T.bool_v, ts_kv_latest.str_v = T.str_v, ts_kv_latest.long_v = T.long_v, ts_kv_latest.dbl_v = T.dbl_v, ts_kv_latest.json_v = T.json_v " + - "WHEN NOT MATCHED THEN INSERT (entity_id, key, ts, bool_v, str_v, long_v, dbl_v) " + - "VALUES (T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v);"; + "WHEN NOT MATCHED THEN INSERT (entity_id, key, ts, bool_v, str_v, long_v, dbl_v, json_v) " + + "VALUES (T.entity_id, T.key, T.ts, T.bool_v, T.str_v, T.long_v, T.dbl_v, T.json_v);"; @Override public void saveOrUpdate(List entities) { diff --git a/dao/src/test/resources/sql/hsql/drop-all-tables.sql b/dao/src/test/resources/sql/hsql/drop-all-tables.sql index 1bdc1a7ece..778a4bbd9a 100644 --- a/dao/src/test/resources/sql/hsql/drop-all-tables.sql +++ b/dao/src/test/resources/sql/hsql/drop-all-tables.sql @@ -13,6 +13,7 @@ DROP TABLE IF EXISTS relation; DROP TABLE IF EXISTS tb_user; DROP TABLE IF EXISTS tenant; DROP TABLE IF EXISTS ts_kv; +DROP TABLE IF EXISTS ts_kv_dictionary; DROP TABLE IF EXISTS ts_kv_latest; DROP TABLE IF EXISTS user_credentials; DROP TABLE IF EXISTS widget_type; From 37940867263ed7cc8bca68187a68e3401460b815 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 28 Aug 2020 15:24:31 +0300 Subject: [PATCH 32/39] added logs for in memory queue --- .../server/queue/memory/InMemoryStorage.java | 18 ++++++++++++++++++ .../queue/memory/InMemoryTbQueueProducer.java | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index f51f5b8ae0..c4414089c4 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -23,16 +23,29 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @Slf4j public final class InMemoryStorage { private static InMemoryStorage instance; private final ConcurrentHashMap> storage; + private static ScheduledExecutorService statExecutor; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); + statExecutor = Executors.newSingleThreadScheduledExecutor(); + statExecutor.scheduleAtFixedRate(this::printStats, 30, 30, TimeUnit.SECONDS); + } + + private void printStats() { + storage.forEach((topic, queue) -> { + if (queue.size() > 0) { + log.debug("Topic: [{}], Queue size: [{}]", topic, queue.size()); + } + }); } public static InMemoryStorage getInstance() { @@ -77,4 +90,9 @@ public final class InMemoryStorage { storage.clear(); } + public void destroy() { + if (statExecutor != null) { + statExecutor.shutdownNow(); + } + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index cfcd788a16..84a9a1fdf0 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -53,6 +53,6 @@ public class InMemoryTbQueueProducer implements TbQueuePro @Override public void stop() { - + storage.destroy(); } } From 17ee07e09a0ae11b18e9aeb280cbd7a79afe3e8b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 28 Aug 2020 16:01:43 +0300 Subject: [PATCH 33/39] changed log time --- .../org/thingsboard/server/queue/memory/InMemoryStorage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index c4414089c4..994ca26305 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -37,7 +37,7 @@ public final class InMemoryStorage { private InMemoryStorage() { storage = new ConcurrentHashMap<>(); statExecutor = Executors.newSingleThreadScheduledExecutor(); - statExecutor.scheduleAtFixedRate(this::printStats, 30, 30, TimeUnit.SECONDS); + statExecutor.scheduleAtFixedRate(this::printStats, 60, 60, TimeUnit.SECONDS); } private void printStats() { From 18ca330c9ea4120c6d8d28d18ad0d1adf13c89e9 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 28 Aug 2020 17:02:04 +0300 Subject: [PATCH 34/39] in memory queue logs --- application/src/main/resources/logback.xml | 2 ++ application/src/main/resources/thingsboard.yml | 4 ++++ .../server/queue/memory/InMemoryStorage.java | 15 ++------------- .../queue/memory/InMemoryTbQueueProducer.java | 2 +- .../provider/InMemoryMonolithQueueFactory.java | 7 +++++++ 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 01aec9ceb9..81eb788e35 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -29,6 +29,8 @@ + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index b6894bb0e7..ac76b67cd3 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -594,6 +594,10 @@ swagger: queue: type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) or aws-sqs (AWS SQS) or pubsub (PubSub) or service-bus (Azure Service Bus) or rabbitmq (RabbitMQ) + in_memory: + stats: + # For debug lvl + print-interval-ms: "${TB_QUEUE_IN_MEMORY_STATS_PRINT_INTERVAL_MS:60000}" kafka: bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" acks: "${TB_KAFKA_ACKS:all}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java index 994ca26305..31f7958bbf 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryStorage.java @@ -23,27 +23,21 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; @Slf4j public final class InMemoryStorage { private static InMemoryStorage instance; private final ConcurrentHashMap> storage; - private static ScheduledExecutorService statExecutor; private InMemoryStorage() { storage = new ConcurrentHashMap<>(); - statExecutor = Executors.newSingleThreadScheduledExecutor(); - statExecutor.scheduleAtFixedRate(this::printStats, 60, 60, TimeUnit.SECONDS); } - private void printStats() { + public void printStats() { storage.forEach((topic, queue) -> { if (queue.size() > 0) { - log.debug("Topic: [{}], Queue size: [{}]", topic, queue.size()); + log.debug("[{}] Queue Size [{}]", topic, queue.size()); } }); } @@ -90,9 +84,4 @@ public final class InMemoryStorage { storage.clear(); } - public void destroy() { - if (statExecutor != null) { - statExecutor.shutdownNow(); - } - } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java index 84a9a1fdf0..cfcd788a16 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/memory/InMemoryTbQueueProducer.java @@ -53,6 +53,6 @@ public class InMemoryTbQueueProducer implements TbQueuePro @Override public void stop() { - storage.destroy(); + } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index fbdbeaab0c..8e2ff01e28 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -17,6 +17,7 @@ package org.thingsboard.server.queue.provider; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.gen.js.JsInvokeProtos; @@ -28,6 +29,7 @@ import org.thingsboard.server.queue.common.TbProtoJsQueueMsg; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; +import org.thingsboard.server.queue.memory.InMemoryStorage; import org.thingsboard.server.queue.memory.InMemoryTbQueueConsumer; import org.thingsboard.server.queue.memory.InMemoryTbQueueProducer; import org.thingsboard.server.queue.settings.TbQueueCoreSettings; @@ -120,4 +122,9 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE public TbQueueRequestTemplate, TbProtoQueueMsg> createRemoteJsRequestTemplate() { return null; } + + @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") + private void printInMemoryStats() { + InMemoryStorage.getInstance().printStats(); + } } From e5ad5e0b301b7434def69817732f84b33d018040 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 28 Aug 2020 17:36:57 +0300 Subject: [PATCH 35/39] refactored --- .../server/queue/provider/InMemoryMonolithQueueFactory.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java index 8e2ff01e28..8ebda6e7b9 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/InMemoryMonolithQueueFactory.java @@ -49,6 +49,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE private final TbQueueRuleEngineSettings ruleEngineSettings; private final TbQueueTransportApiSettings transportApiSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final InMemoryStorage storage; public InMemoryMonolithQueueFactory(PartitionService partitionService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -61,6 +62,7 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE this.ruleEngineSettings = ruleEngineSettings; this.transportApiSettings = transportApiSettings; this.transportNotificationSettings = transportNotificationSettings; + this.storage = InMemoryStorage.getInstance(); } @Override @@ -125,6 +127,6 @@ public class InMemoryMonolithQueueFactory implements TbCoreQueueFactory, TbRuleE @Scheduled(fixedRateString = "${queue.in_memory.stats.print-interval-ms:60000}") private void printInMemoryStats() { - InMemoryStorage.getInstance().printStats(); + storage.printStats(); } } From 8ec23f22d4a5443108da4cc326fb7fc43a9ce52e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 28 Aug 2020 19:02:31 +0300 Subject: [PATCH 36/39] Version set 2.5.5-SNAPSHOT --- application/pom.xml | 2 +- common/actor/pom.xml | 2 +- common/dao-api/pom.xml | 2 +- common/data/pom.xml | 2 +- common/message/pom.xml | 2 +- common/pom.xml | 2 +- common/queue/pom.xml | 2 +- common/stats/pom.xml | 2 +- common/transport/coap/pom.xml | 2 +- common/transport/http/pom.xml | 2 +- common/transport/mqtt/pom.xml | 2 +- common/transport/pom.xml | 2 +- common/transport/transport-api/pom.xml | 2 +- common/util/pom.xml | 2 +- dao/pom.xml | 2 +- msa/black-box-tests/pom.xml | 2 +- msa/js-executor/package.json | 2 +- msa/js-executor/pom.xml | 2 +- msa/pom.xml | 2 +- msa/tb-node/pom.xml | 2 +- msa/tb/pom.xml | 2 +- msa/transport/coap/pom.xml | 2 +- msa/transport/http/pom.xml | 2 +- msa/transport/mqtt/pom.xml | 2 +- msa/transport/pom.xml | 2 +- msa/web-ui/package.json | 2 +- msa/web-ui/pom.xml | 2 +- netty-mqtt/pom.xml | 4 ++-- pom.xml | 2 +- rest-client/pom.xml | 2 +- rule-engine/pom.xml | 2 +- rule-engine/rule-engine-api/pom.xml | 2 +- rule-engine/rule-engine-components/pom.xml | 2 +- tools/pom.xml | 2 +- transport/coap/pom.xml | 2 +- transport/http/pom.xml | 2 +- transport/mqtt/pom.xml | 2 +- transport/pom.xml | 2 +- ui/package.json | 2 +- ui/pom.xml | 2 +- 40 files changed, 41 insertions(+), 41 deletions(-) diff --git a/application/pom.xml b/application/pom.xml index ffb60b169a..1730d7a5a5 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard application diff --git a/common/actor/pom.xml b/common/actor/pom.xml index 542715c2f4..ee626dac6f 100644 --- a/common/actor/pom.xml +++ b/common/actor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/common/dao-api/pom.xml b/common/dao-api/pom.xml index 204438477f..879e479c3a 100644 --- a/common/dao-api/pom.xml +++ b/common/dao-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/common/data/pom.xml b/common/data/pom.xml index acd856e14e..8b43e5c8b1 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/common/message/pom.xml b/common/message/pom.xml index 2df71c6332..62802d141c 100644 --- a/common/message/pom.xml +++ b/common/message/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/common/pom.xml b/common/pom.xml index 67f43bf6a6..e0a3865e08 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard common diff --git a/common/queue/pom.xml b/common/queue/pom.xml index f2ebc7ae54..b302313baf 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/common/stats/pom.xml b/common/stats/pom.xml index 7a38c27c2a..8ccf34961c 100644 --- a/common/stats/pom.xml +++ b/common/stats/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/coap/pom.xml b/common/transport/coap/pom.xml index 1e1753a0cc..a237662b0d 100644 --- a/common/transport/coap/pom.xml +++ b/common/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/http/pom.xml b/common/transport/http/pom.xml index 8a4bb46cd9..715cb3d169 100644 --- a/common/transport/http/pom.xml +++ b/common/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/mqtt/pom.xml b/common/transport/mqtt/pom.xml index ccf1f7f804..e3579876e4 100644 --- a/common/transport/mqtt/pom.xml +++ b/common/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/pom.xml b/common/transport/pom.xml index b5e89fe709..e40f3e4009 100644 --- a/common/transport/pom.xml +++ b/common/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index b1d8c6b7bc..739aa2b068 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/util/pom.xml b/common/util/pom.xml index 317c9a061e..fceed3ee0e 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT common org.thingsboard.common diff --git a/dao/pom.xml b/dao/pom.xml index 479eecf94f..8c1cd17bf4 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard dao diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 1e2b5e26b3..d24f38bc36 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 599968a367..967f55b24f 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-js-executor", "private": true, - "version": "2.5.4", + "version": "2.5.5", "description": "ThingsBoard JavaScript Executor Microservice", "main": "server.js", "bin": "server.js", diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index 12a1e33de5..60ccd9d5bb 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/pom.xml b/msa/pom.xml index 4221d1c0ea..c627dca51a 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard msa diff --git a/msa/tb-node/pom.xml b/msa/tb-node/pom.xml index 575beeeb08..944521f87e 100644 --- a/msa/tb-node/pom.xml +++ b/msa/tb-node/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/tb/pom.xml b/msa/tb/pom.xml index da384db8df..349cc36337 100644 --- a/msa/tb/pom.xml +++ b/msa/tb/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/transport/coap/pom.xml b/msa/transport/coap/pom.xml index 3575a36f9b..3e956736a1 100644 --- a/msa/transport/coap/pom.xml +++ b/msa/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/http/pom.xml b/msa/transport/http/pom.xml index bee5188f63..bc8ccc3f1f 100644 --- a/msa/transport/http/pom.xml +++ b/msa/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/mqtt/pom.xml b/msa/transport/mqtt/pom.xml index 5eeb52b724..b2bc3ff8d9 100644 --- a/msa/transport/mqtt/pom.xml +++ b/msa/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.msa.transport diff --git a/msa/transport/pom.xml b/msa/transport/pom.xml index f45a68e2fd..d00683c492 100644 --- a/msa/transport/pom.xml +++ b/msa/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index acbe02cafb..2eec7556f9 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-web-ui", "private": true, - "version": "2.5.4", + "version": "2.5.5", "description": "ThingsBoard Web UI Microservice", "main": "server.js", "bin": "server.js", diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index aeabf26ab9..660920835b 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT msa org.thingsboard.msa diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index 701d128380..58cbcf4e6a 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -19,11 +19,11 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard netty-mqtt - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT jar Netty MQTT Client diff --git a/pom.xml b/pom.xml index 640609c78b..335e6c7eba 100755 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT pom Thingsboard diff --git a/rest-client/pom.xml b/rest-client/pom.xml index bc60de59ed..061efa390b 100644 --- a/rest-client/pom.xml +++ b/rest-client/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard rest-client diff --git a/rule-engine/pom.xml b/rule-engine/pom.xml index d73c3dfe78..00d445cc3c 100644 --- a/rule-engine/pom.xml +++ b/rule-engine/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard rule-engine diff --git a/rule-engine/rule-engine-api/pom.xml b/rule-engine/rule-engine-api/pom.xml index 1bd34eb2d3..2c1e323165 100644 --- a/rule-engine/rule-engine-api/pom.xml +++ b/rule-engine/rule-engine-api/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index f38fe14d22..00d8101e5a 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/tools/pom.xml b/tools/pom.xml index 533f802c2e..91bf319b0e 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard tools diff --git a/transport/coap/pom.xml b/transport/coap/pom.xml index a6d69e20ed..eeea5a378a 100644 --- a/transport/coap/pom.xml +++ b/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/http/pom.xml b/transport/http/pom.xml index 4869f01fe2..7540b78507 100644 --- a/transport/http/pom.xml +++ b/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/mqtt/pom.xml b/transport/mqtt/pom.xml index 141d0ae6b0..6f4edd87b1 100644 --- a/transport/mqtt/pom.xml +++ b/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT transport org.thingsboard.transport diff --git a/transport/pom.xml b/transport/pom.xml index 5b4cbec2a2..59b5b2470b 100644 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard transport diff --git a/ui/package.json b/ui/package.json index e3691c5ef8..cbc4b8c6af 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard", "private": true, - "version": "2.5.4", + "version": "2.5.5", "description": "ThingsBoard UI", "licenses": [ { diff --git a/ui/pom.xml b/ui/pom.xml index 3bf9eced54..b27f492c97 100644 --- a/ui/pom.xml +++ b/ui/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 2.5.4-SNAPSHOT + 2.5.5-SNAPSHOT thingsboard org.thingsboard From bc2520fbbb95d06e63dea73f5c5d47fc08ce5aee Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 1 Sep 2020 14:53:47 +0300 Subject: [PATCH 37/39] add request param "orderBy" for getTimeseries --- .../thingsboard/server/controller/TelemetryController.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java index bb866e3e9b..af4eb0f08f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TelemetryController.java @@ -197,19 +197,21 @@ public class TelemetryController extends BaseController { @RequestMapping(value = "/{entityType}/{entityId}/values/timeseries", method = RequestMethod.GET, params = {"keys", "startTs", "endTs"}) @ResponseBody public DeferredResult getTimeseries( - @PathVariable("entityType") String entityType, @PathVariable("entityId") String entityIdStr, + @PathVariable("entityType") String entityType, + @PathVariable("entityId") String entityIdStr, @RequestParam(name = "keys") String keys, @RequestParam(name = "startTs") Long startTs, @RequestParam(name = "endTs") Long endTs, @RequestParam(name = "interval", defaultValue = "0") Long interval, @RequestParam(name = "limit", defaultValue = "100") Integer limit, @RequestParam(name = "agg", defaultValue = "NONE") String aggStr, + @RequestParam(name= "orderBy", defaultValue = "DESC") String orderBy, @RequestParam(name = "useStrictDataTypes", required = false, defaultValue = "false") Boolean useStrictDataTypes) throws ThingsboardException { return accessValidator.validateEntityAndCallback(getCurrentUser(), Operation.READ_TELEMETRY, entityType, entityIdStr, (result, tenantId, entityId) -> { // If interval is 0, convert this to a NONE aggregation, which is probably what the user really wanted Aggregation agg = interval == 0L ? Aggregation.valueOf(Aggregation.NONE.name()) : Aggregation.valueOf(aggStr); - List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg)) + List queries = toKeysList(keys).stream().map(key -> new BaseReadTsKvQuery(key, startTs, endTs, interval, limit, agg, orderBy)) .collect(Collectors.toList()); Futures.addCallback(tsService.findAll(tenantId, entityId, queries), getTsKvListCallback(result, useStrictDataTypes), MoreExecutors.directExecutor()); From 34dbc81de6890a37a28043377f0a10d3ffab54de Mon Sep 17 00:00:00 2001 From: Dmytro Shvaika Date: Tue, 1 Sep 2020 15:47:06 +0300 Subject: [PATCH 38/39] fix drop partitions by max ttl procedure --- .../schema_update_psql_drop_partitions.sql | 1 + .../install/ThingsboardInstallService.java | 5 +++++ .../CassandraTsDatabaseUpgradeService.java | 1 + .../install/PsqlTsDatabaseUpgradeService.java | 6 ++++++ .../TimescaleTsDatabaseUpgradeService.java | 2 ++ .../service/ttl/AbstractCleanUpService.java | 18 +++++++----------- .../ttl/events/EventsCleanUpService.java | 2 +- .../PsqlTimeseriesCleanUpService.java | 3 ++- .../TimescaleTimeseriesCleanUpService.java | 3 ++- dao/src/main/resources/sql/schema-ts-psql.sql | 1 + 10 files changed, 28 insertions(+), 14 deletions(-) diff --git a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql index 0916c241a1..41e1cfbb7a 100644 --- a/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql +++ b/application/src/main/data/upgrade/2.4.3/schema_update_psql_drop_partitions.sql @@ -64,6 +64,7 @@ BEGIN AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' AND tablename != 'ts_kv_dictionary' + AND tablename != 'ts_kv_indefinite' LOOP IF partition != partition_by_max_ttl_date THEN IF partition_year IS NOT NULL THEN diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index e281c0958e..01ad8a29e9 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -146,6 +146,11 @@ public class ThingsboardInstallService { databaseTsUpgradeService.upgradeDatabase("2.5.0"); } + case "2.5.4": + log.info("Upgrading ThingsBoard from version 2.5.4 to 2.5.5 ..."); + if (databaseTsUpgradeService != null) { + databaseTsUpgradeService.upgradeDatabase("2.5.4"); + } log.info("Updating system data..."); diff --git a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java index 103e8090d9..07b8522323 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/CassandraTsDatabaseUpgradeService.java @@ -49,6 +49,7 @@ public class CassandraTsDatabaseUpgradeService extends AbstractCassandraDatabase log.info("Schema updated."); break; case "2.5.0": + case "2.5.4": break; default: throw new RuntimeException("Unable to upgrade Cassandra database, unsupported fromVersion: " + fromVersion); diff --git a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java index 7a8174af16..396f84664a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/PsqlTsDatabaseUpgradeService.java @@ -195,6 +195,12 @@ public class PsqlTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgradeSe executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001"); } break; + case "2.5.4": + try (Connection conn = DriverManager.getConnection(dbUrl, dbUserName, dbPassword)) { + log.info("Load Drop Partitions functions ..."); + loadSql(conn, LOAD_DROP_PARTITIONS_FUNCTIONS_SQL); + } + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java index d8f7ea61f9..a929a51fb5 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/TimescaleTsDatabaseUpgradeService.java @@ -177,6 +177,8 @@ public class TimescaleTsDatabaseUpgradeService extends AbstractSqlTsDatabaseUpgr executeQuery(conn, "UPDATE tb_schema_settings SET schema_version = 2005001"); } break; + case "2.5.4": + break; default: throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 4fc4df0048..e81788958d 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -38,19 +38,15 @@ public abstract class AbstractCleanUpService { @Value("${spring.datasource.password}") protected String dbPassword; - protected long executeQuery(Connection conn, String query) { - long removed = 0L; - try { - Statement statement = conn.createStatement(); + protected long executeQuery(Connection conn, String query) throws SQLException { + try (Statement statement = conn.createStatement()) { ResultSet resultSet = statement.executeQuery(query); - getWarnings(statement); + if (log.isDebugEnabled()) { + getWarnings(statement); + } resultSet.next(); - removed = resultSet.getLong(1); - log.debug("Successfully executed query: {}", query); - } catch (SQLException e) { - log.debug("Failed to execute query: {} due to: {}", query, e.getMessage()); + return resultSet.getLong(1); } - return removed; } protected void getWarnings(Statement statement) throws SQLException { @@ -65,6 +61,6 @@ public abstract class AbstractCleanUpService { } } - protected abstract void doCleanUp(Connection connection); + protected abstract void doCleanUp(Connection connection) throws SQLException; } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java index 5b094c5c0e..ca52bca7e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/events/EventsCleanUpService.java @@ -54,7 +54,7 @@ public class EventsCleanUpService extends AbstractCleanUpService { } @Override - protected void doCleanUp(Connection connection) { + protected void doCleanUp(Connection connection) throws SQLException { long totalEventsRemoved = executeQuery(connection, "call cleanup_events_by_ttl(" + ttl + ", " + debugTtl + ", 0);"); log.info("Total events removed by TTL: [{}]", totalEventsRemoved); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java index cd403ee3b8..2464ab4677 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/PsqlTimeseriesCleanUpService.java @@ -22,6 +22,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.PsqlTsDao; import java.sql.Connection; +import java.sql.SQLException; @PsqlTsDao @Service @@ -32,7 +33,7 @@ public class PsqlTimeseriesCleanUpService extends AbstractTimeseriesCleanUpServi private String partitionType; @Override - protected void doCleanUp(Connection connection) { + protected void doCleanUp(Connection connection) throws SQLException { long totalPartitionsRemoved = executeQuery(connection, "call drop_partitions_by_max_ttl('" + partitionType + "'," + systemTtl + ", 0);"); log.info("Total partitions removed by TTL: [{}]", totalPartitionsRemoved); long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);"); diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java index f5898b9b20..8bdeea46ae 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/timeseries/TimescaleTimeseriesCleanUpService.java @@ -21,6 +21,7 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.util.TimescaleDBTsDao; import java.sql.Connection; +import java.sql.SQLException; @TimescaleDBTsDao @Service @@ -28,7 +29,7 @@ import java.sql.Connection; public class TimescaleTimeseriesCleanUpService extends AbstractTimeseriesCleanUpService { @Override - protected void doCleanUp(Connection connection) { + protected void doCleanUp(Connection connection) throws SQLException { long totalEntitiesTelemetryRemoved = executeQuery(connection, "call cleanup_timeseries_by_ttl('" + ModelConstants.NULL_UUID_STR + "'," + systemTtl + ", 0);"); log.info("Total telemetry removed stats by TTL for entities: [{}]", totalEntitiesTelemetryRemoved); } diff --git a/dao/src/main/resources/sql/schema-ts-psql.sql b/dao/src/main/resources/sql/schema-ts-psql.sql index 28420a8957..80f111869d 100644 --- a/dao/src/main/resources/sql/schema-ts-psql.sql +++ b/dao/src/main/resources/sql/schema-ts-psql.sql @@ -105,6 +105,7 @@ BEGIN AND tablename like 'ts_kv_' || '%' AND tablename != 'ts_kv_latest' AND tablename != 'ts_kv_dictionary' + AND tablename != 'ts_kv_indefinite' LOOP IF partition != partition_by_max_ttl_date THEN IF partition_year IS NOT NULL THEN From f16672cbf69717f8506937ec20a87dee4d7052f3 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 2 Sep 2020 18:12:08 +0300 Subject: [PATCH 39/39] added partition property for kafka --- application/src/main/resources/thingsboard.yml | 10 +++++----- .../server/queue/kafka/TbKafkaAdmin.java | 10 +++++++++- docker/queue-kafka.env | 1 + msa/js-executor/config/default.yml | 2 +- msa/js-executor/queue/kafkaTemplate.js | 18 +++++++++++++++--- .../src/main/resources/tb-coap-transport.yml | 10 +++++----- .../src/main/resources/tb-http-transport.yml | 10 +++++----- .../src/main/resources/tb-mqtt-transport.yml | 10 +++++----- 8 files changed, 46 insertions(+), 25 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index ac76b67cd3..6400a6bbf0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -617,11 +617,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java index b92b094af1..bee293bd6d 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaAdmin.java @@ -37,6 +37,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { private final AdminClient client; private final Map topicConfigs; private final Set topics = ConcurrentHashMap.newKeySet(); + private final int numPartitions; private final short replicationFactor; @@ -50,6 +51,13 @@ public class TbKafkaAdmin implements TbQueueAdmin { log.error("Failed to get all topics.", e); } + String numPartitionsStr = topicConfigs.get("partitions"); + if (numPartitionsStr != null) { + numPartitions = Integer.parseInt(numPartitionsStr); + topicConfigs.remove("partitions"); + } else { + numPartitions = 1; + } replicationFactor = settings.getReplicationFactor(); } @@ -59,7 +67,7 @@ public class TbKafkaAdmin implements TbQueueAdmin { return; } try { - NewTopic newTopic = new NewTopic(topic, 1, replicationFactor).configs(topicConfigs); + NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor).configs(topicConfigs); createTopic(newTopic).values().get(topic).get(); topics.add(topic); } catch (ExecutionException ee) { diff --git a/docker/queue-kafka.env b/docker/queue-kafka.env index 63107942fb..0207d64ef5 100644 --- a/docker/queue-kafka.env +++ b/docker/queue-kafka.env @@ -1,2 +1,3 @@ TB_QUEUE_TYPE=kafka TB_KAFKA_SERVERS=kafka:9092 +TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES=retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100 diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index fac1ac7e8a..e6ec6f5b36 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -25,7 +25,7 @@ kafka: # Kafka Bootstrap Servers servers: "localhost:9092" replication_factor: "1" - topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600" + topic_properties: "retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100" use_confluent_cloud: false confluent: sasl: diff --git a/msa/js-executor/queue/kafkaTemplate.js b/msa/js-executor/queue/kafkaTemplate.js index 0420173188..2672a8df6a 100644 --- a/msa/js-executor/queue/kafkaTemplate.js +++ b/msa/js-executor/queue/kafkaTemplate.js @@ -34,7 +34,7 @@ function KafkaProducer() { this.send = async (responseTopic, scriptId, rawResponse, headers) => { if (!topics.includes(responseTopic)) { - let createResponseTopicResult = await createTopic(responseTopic); + let createResponseTopicResult = await createTopic(responseTopic, 1); topics.push(responseTopic); if (createResponseTopicResult) { logger.info('Created new topic: %s', requestTopic); @@ -88,7 +88,18 @@ function KafkaProducer() { kafkaAdmin = kafkaClient.admin(); await kafkaAdmin.connect(); - let createRequestTopicResult = await createTopic(requestTopic); + let partitions = 1; + + for (let i = 0; i < configEntries.length; i++) { + let param = configEntries[i]; + if (param.name === 'partitions') { + partitions = param.value; + configEntries.splice(i, 1); + break; + } + } + + let createRequestTopicResult = await createTopic(requestTopic, partitions); if (createRequestTopicResult) { logger.info('Created new topic: %s', requestTopic); @@ -121,10 +132,11 @@ function KafkaProducer() { } })(); -function createTopic(topic) { +function createTopic(topic, partitions) { return kafkaAdmin.createTopics({ topics: [{ topic: topic, + numPartitions: partitions, replicationFactor: replicationFactor, configEntries: configEntries }] diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index f04a7559b8..3f57b32fab 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -77,11 +77,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index a97a58cd85..77d5f30fa7 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -70,11 +70,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index b3804e3531..bc18112c8e 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -98,11 +98,11 @@ queue: security.protocol: "${TB_QUEUE_KAFKA_CONFLUENT_SECURITY_PROTOCOL:SASL_SSL}" other: topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600}" + rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" + js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}"