From 24a5ed2e29dc4586b91c78b9f54762fa62e83855 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 29 Jun 2022 15:15:57 +0300 Subject: [PATCH 01/28] black-box tests memory limit --- .../java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index b627d2787a..c39932b572 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -46,6 +46,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; + private final static String JAVA_OPTS = "-Xmx384m -Xss256k"; private final DockerComposeExecutor dockerCompose; @@ -102,6 +103,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose = new DockerComposeExecutor(composeFiles, project); env = new HashMap<>(); + env.put("JAVA_OPTS", JAVA_OPTS); env.put("POSTGRES_DATA_VOLUME", postgresDataVolume); if (IS_HYBRID_MODE) { env.put("CASSANDRA_DATA_VOLUME", cassandraDataVolume); From 74e465bf58b6ad64892ee3c6e5fb0518ad29f787 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 29 Jun 2022 15:17:53 +0300 Subject: [PATCH 02/28] docker-compose: JAVA_OPTS env added for each Java service --- docker/.env | 5 ++++- docker/docker-compose.yml | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/docker/.env b/docker/.env index 7bc1a7a6a4..f33ed50da5 100644 --- a/docker/.env +++ b/docker/.env @@ -25,4 +25,7 @@ DATABASE=postgres LOAD_BALANCER_NAME=haproxy-certbot # If enabled Prometheus and Grafana containers are deployed along with other containers -MONITORING_ENABLED=false \ No newline at end of file +MONITORING_ENABLED=false + +# Limit memory usage for each Java application +# JAVA_OPTS=-Xmx2048M -Xms2048M -Xss384k -XX:+AlwaysPreTouch diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 3e0f21391d..6ec410fcde 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -48,6 +48,7 @@ services: TB_SERVICE_ID: tb-core1 TB_SERVICE_TYPE: tb-core EDGES_ENABLED: "true" + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-node.env volumes: @@ -73,6 +74,7 @@ services: TB_SERVICE_ID: tb-core2 TB_SERVICE_TYPE: tb-core EDGES_ENABLED: "true" + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-node.env volumes: @@ -96,6 +98,7 @@ services: environment: TB_SERVICE_ID: tb-rule-engine1 TB_SERVICE_TYPE: tb-rule-engine + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-node.env volumes: @@ -117,6 +120,7 @@ services: environment: TB_SERVICE_ID: tb-rule-engine2 TB_SERVICE_TYPE: tb-rule-engine + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-node.env volumes: @@ -132,6 +136,7 @@ services: - "1883" environment: TB_SERVICE_ID: tb-mqtt-transport1 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-mqtt-transport.env volumes: @@ -148,6 +153,7 @@ services: - "1883" environment: TB_SERVICE_ID: tb-mqtt-transport2 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-mqtt-transport.env volumes: @@ -164,6 +170,7 @@ services: - "8081" environment: TB_SERVICE_ID: tb-http-transport1 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-http-transport.env volumes: @@ -180,6 +187,7 @@ services: - "8081" environment: TB_SERVICE_ID: tb-http-transport2 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-http-transport.env volumes: @@ -196,6 +204,7 @@ services: - "5683:5683/udp" environment: TB_SERVICE_ID: tb-coap-transport + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-coap-transport.env volumes: @@ -212,6 +221,7 @@ services: - "5685:5685/udp" environment: TB_SERVICE_ID: tb-lwm2m-transport + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-lwm2m-transport.env volumes: @@ -226,6 +236,7 @@ services: image: "${DOCKER_REPO}/${SNMP_TRANSPORT_DOCKER_NAME}:${TB_VERSION}" environment: TB_SERVICE_ID: tb-snmp-transport + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-snmp-transport.env volumes: @@ -256,6 +267,7 @@ services: - "8081" environment: TB_SERVICE_ID: tb-vc-executor1 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-vc-executor.env volumes: @@ -272,6 +284,7 @@ services: - "8081" environment: TB_SERVICE_ID: tb-vc-executor2 + JAVA_OPTS: "${JAVA_OPTS}" env_file: - tb-vc-executor.env volumes: From b1ee3415a62fc072c6f0ed1685b2d04c3518a974 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 5 Jul 2022 12:07:57 +0300 Subject: [PATCH 03/28] refactoring: tests EntityView --- .../controller/AbstractNotifyEntityTest.java | 68 +++++--- .../controller/BaseDeviceControllerTest.java | 4 +- .../BaseEntityRelationControllerTest.java | 2 - .../BaseEntityViewControllerTest.java | 149 +++++++++++++++--- 4 files changed, 171 insertions(+), 52 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index a5af165699..b5c26e006c 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -69,7 +69,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { int cntTime = 1; - testSendNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); + testNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); @@ -96,8 +96,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } protected void testNotifyEntityAllManyRelation(EntityRelation relation, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType, int cntTime) { + TenantId tenantId, CustomerId customerId, UserId userId, String userName, + ActionType actionType, int cntTime) { Mockito.verify(tbClusterService, times(cntTime)).sendNotificationMsgToEdge(Mockito.eq(tenantId), Mockito.isNull(), Mockito.isNull(), Mockito.any(), Mockito.eq(EdgeEventType.RELATION), Mockito.eq(edgeTypeByActionType(actionType))); @@ -113,7 +113,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { int cntTime = 1; - testSendNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); + testNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); testLogEntityActionEntityEqClass(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); @@ -122,7 +122,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { protected void testNotifyEntityNeverMsgToEdgeServiceOneTime(HasName entity, EntityId entityId, TenantId tenantId, ActionType actionType) { - testSendNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, 1); + testNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, 1); testLogEntityActionNever(entityId, entity); testPushMsgToRuleEngineNever(entityId); Mockito.reset(tbClusterService, auditLogService); @@ -132,7 +132,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { int cntTime = 1; - testNotificationMsgToEdgeServiceNever(entityId); + testNotificationMsgToEdgeServiceNeverWithActionType(entityId, actionType); testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); if (ActionType.RELATIONS_DELETED.equals(actionType)) { @@ -148,7 +148,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { ActionType actionType, int cntTime, Object... additionalInfo) { EntityId entityId = createEntityId_NULL_UUID(entity); EntityId originatorId = createEntityId_NULL_UUID(originator); - testNotificationMsgToEdgeServiceNever(entityId); + testNotificationMsgToEdgeServiceNeverWithActionType(entityId, actionType); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, @@ -189,7 +189,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { ActionType actionType, int cntTime, int cntAdditionalInfo) { EntityId entityId = createEntityId_NULL_UUID(entity); EntityId originatorId = createEntityId_NULL_UUID(originator); - testNotificationMsgToEdgeServiceNever(entityId); + testNotificationMsgToEdgeServiceNeverWithActionType(entityId, actionType); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, @@ -202,7 +202,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { int cntTime = 1; - testSendNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); + testNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); @@ -214,7 +214,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { int cntTime = 1; - testNotificationMsgToEdgeServiceNever(entityId); + testNotificationMsgToEdgeServiceNeverWithActionType(entityId, actionType); testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); @@ -222,26 +222,28 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.reset(tbClusterService, auditLogService); } - protected void testNotifyEntityBroadcastEntityStateChangeEventManyMsgToEdgeServiceNever(HasName entity, HasName originator, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType, int cntTime, int cntAdditionalInfo) { + protected void testNotifyEntityBroadcastEntityStateChangeEventMany(HasName entity, HasName originator, + TenantId tenantId, CustomerId customerId, + UserId userId, String userName, ActionType actionType, + ActionType actionTypeEdge, + int cntTime, int cntTimeEdge, int cntTimeRuleEngine, + int cntAdditionalInfo) { EntityId entityId = createEntityId_NULL_UUID(entity); EntityId originatorId = createEntityId_NULL_UUID(originator); - testNotificationMsgToEdgeServiceNever(entityId); + testNotificationMsgToEdgeServiceTime(entityId, tenantId, actionTypeEdge, cntTimeEdge); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, cntAdditionalInfo); - testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); + testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTimeRuleEngine); testBroadcastEntityStateChangeEventTime(entityId, tenantId, cntTime); - Mockito.reset(tbClusterService, auditLogService); } protected void testNotifyEntityMsgToEdgePushMsgToCoreOneTime(HasName entity, EntityId entityId, EntityId originatorId, TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, Object... additionalInfo) { int cntTime = 1; - testSendNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); + testNotificationMsgToEdgeServiceTime(entityId, tenantId, actionType, cntTime); testLogEntityAction(entity, originatorId, tenantId, customerId, userId, userName, actionType, cntTime, additionalInfo); tesPushMsgToCoreTime(cntTime); Mockito.reset(tbClusterService, auditLogService); @@ -252,7 +254,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Object... additionalInfo) { CustomerId customer_NULL_UUID = (CustomerId) EntityIdFactory.getByTypeAndUuid(EntityType.CUSTOMER, ModelConstants.NULL_UUID); EntityId entity_originator_NULL_UUID = createEntityId_NULL_UUID(entity); - testNotificationMsgToEdgeServiceNever(entity_originator_NULL_UUID); + testNotificationMsgToEdgeServiceNeverWithActionType(entity_originator_NULL_UUID, actionType); ArgumentMatcher matcherEntityEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherError = argument -> argument.getMessage().contains(exp.getMessage()) & argument.getClass().equals(exp.getClass()); @@ -266,7 +268,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Object... additionalInfo) { CustomerId customer_NULL_UUID = (CustomerId) EntityIdFactory.getByTypeAndUuid(EntityType.CUSTOMER, ModelConstants.NULL_UUID); EntityId entity_originator_NULL_UUID = createEntityId_NULL_UUID(entity); - testNotificationMsgToEdgeServiceNever(entity_originator_NULL_UUID); + testNotificationMsgToEdgeServiceNeverWithActionType(entity_originator_NULL_UUID, actionType); ArgumentMatcher matcherEntityIsNull = Objects::isNull; ArgumentMatcher matcherError = argument -> argument.getMessage().contains(exp.getMessage()) & argument.getClass().equals(exp.getClass()); @@ -299,6 +301,13 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.verify(gatewayNotificationsService, never()).onDeviceDeleted(Mockito.any(Device.class)); } + private void testNotificationMsgToEdgeServiceNeverWithActionType(EntityId entityId, ActionType actionType) { + EdgeEventActionType edgeEventActionType = ActionType.CREDENTIALS_UPDATED.equals(actionType) ? + EdgeEventActionType.CREDENTIALS_UPDATED : edgeTypeByActionType(actionType); + Mockito.verify(tbClusterService, never()).sendNotificationMsgToEdge(Mockito.any(), + Mockito.any(), Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any(), Mockito.eq(edgeEventActionType)); + } + private void testNotificationMsgToEdgeServiceNever(EntityId entityId) { Mockito.verify(tbClusterService, never()).sendNotificationMsgToEdge(Mockito.any(), Mockito.any(), Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any(), Mockito.any()); @@ -317,16 +326,23 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.any(entityId.getClass()), Mockito.any(), Mockito.any()); } + protected void testBroadcastEntityStateChangeEventNever(EntityId entityId) { + Mockito.verify(tbClusterService, never()).broadcastEntityStateChangeEvent(Mockito.any(), + Mockito.any(entityId.getClass()), Mockito.any(ComponentLifecycleEvent.class)); + } + private void testPushMsgToRuleEngineTime(ArgumentMatcher matcherOriginatorId, TenantId tenantId, int cntTime) { Mockito.verify(tbClusterService, times(cntTime)).pushMsgToRuleEngine(Mockito.eq(tenantId), Mockito.argThat(matcherOriginatorId), Mockito.any(TbMsg.class), Mockito.isNull()); } - private void testSendNotificationMsgToEdgeServiceTime(EntityId entityId, TenantId tenantId, ActionType actionType, int cntTime) { + private void testNotificationMsgToEdgeServiceTime(EntityId entityId, TenantId tenantId, ActionType actionType, int cntTime) { EdgeEventActionType edgeEventActionType = ActionType.CREDENTIALS_UPDATED.equals(actionType) ? EdgeEventActionType.CREDENTIALS_UPDATED : edgeTypeByActionType(actionType); + ArgumentMatcher matcherEntityId = cntTime == 1 ? argument -> argument.equals(entityId) : + argument -> argument.getClass().equals(entityId.getClass()); Mockito.verify(tbClusterService, times(cntTime)).sendNotificationMsgToEdge(Mockito.eq(tenantId), - Mockito.any(), Mockito.eq(entityId), Mockito.any(), Mockito.isNull(), + Mockito.any(), Mockito.argThat(matcherEntityId), Mockito.any(), Mockito.isNull(), Mockito.eq(edgeEventActionType)); } @@ -336,7 +352,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.eq(edgeTypeByActionType(actionType))); } - private void testBroadcastEntityStateChangeEventTime(EntityId entityId, TenantId tenantId, int cntTime) { + protected void testBroadcastEntityStateChangeEventTime(EntityId entityId, TenantId tenantId, int cntTime) { Mockito.verify(tbClusterService, times(cntTime)).broadcastEntityStateChangeEvent(Mockito.eq(tenantId), Mockito.any(entityId.getClass()), Mockito.any(ComponentLifecycleEvent.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index dc5f9efa23..273ec4f82f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -113,6 +113,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { Device savedDevice = doPost("/api/device", device, Device.class); Device oldDevice = new Device(savedDevice); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedDevice, savedDevice.getId(), savedDevice.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED); @@ -248,8 +249,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { } testNotifyManyEntityManyTimeMsgToEdgeServiceNever(new Device(), new Device(), - savedTenant.getId(), - tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, cntEntity); testNotificationUpdateGatewayNever(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java index 08a157484f..f78fd6ce8b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java @@ -357,7 +357,6 @@ public abstract class BaseEntityRelationControllerTest extends AbstractControlle .andExpect(statusReason(containsString("Requested item wasn't found!"))); testNotifyEntityNever(mainDevice.getId(), null); - } @Test @@ -570,7 +569,6 @@ public abstract class BaseEntityRelationControllerTest extends AbstractControlle .andExpect(status().isNotFound()) .andExpect(statusReason(containsString(msgErrorNoFound("Device", relation.getFrom().getId().toString())))); - deleteDifferentTenant(); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 615914c6d7..84e617f3fa 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -32,6 +32,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -41,6 +42,7 @@ import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EntityViewInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.objects.AttributesEntityView; @@ -52,6 +54,7 @@ import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.model.ModelConstants; import java.util.ArrayList; @@ -92,7 +95,6 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes @Before public void beforeTest() throws Exception { - log.debug("beforeTest"); executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(8, getClass())); loginTenantAdmin(); @@ -126,6 +128,9 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes @Test public void testSaveEntityView() throws Exception { String name = "Test entity view"; + + Mockito.reset(tbClusterService, auditLogService); + EntityView savedView = getNewSavedEntityView(name); Assert.assertNotNull(savedView); @@ -140,6 +145,10 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes assertEquals(savedView, foundEntityView); + testNotifyEntityBroadcastEntityStateChangeEventOneTimeMsgToEdgeServiceNever(foundEntityView, foundEntityView.getId(), foundEntityView.getId(), + tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.ADDED); + savedView.setName("New test entity view"); doPost("/api/entityView", savedView, EntityView.class); @@ -147,23 +156,57 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes assertEquals(savedView, foundEntityView); - doGet("/api/tenant/entityViews?entityViewName=" + name, EntityView.class, status().isNotFound()); + testBroadcastEntityStateChangeEventTime(foundEntityView.getId(), tenantId, 1); + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundEntityView, foundEntityView, + tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.UPDATED, ActionType.UPDATED, 1, 1, 5); + + doGet("/api/tenant/entityViews?entityViewName=" + name) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString("Requested item wasn't found!"))); } @Test public void testSaveEntityViewWithViolationOfValidation() throws Exception { EntityView entityView = createEntityView(RandomStringUtils.randomAlphabetic(300), 0, 0); - doPost("/api/entityView", entityView).andExpect(statusReason(containsString("length of name must be equal or less than 255"))); + + Mockito.reset(tbClusterService, auditLogService); + + String msgError = msgErrorFieldLength("name"); + doPost("/api/entityView", entityView) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeServiceNeverError(entityView, + tenantId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.ADDED, new DataValidationException(msgError)); + Mockito.reset(tbClusterService, auditLogService); + entityView.setName("Normal name"); + msgError = msgErrorFieldLength("type"); entityView.setType(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/entityView", entityView).andExpect(statusReason(containsString("length of type must be equal or less than 255"))); + doPost("/api/entityView", entityView) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeServiceNeverError(entityView, + tenantId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.ADDED, new DataValidationException(msgError)); } @Test public void testUpdateEntityViewFromDifferentTenant() throws Exception { EntityView savedView = getNewSavedEntityView("Test entity view"); loginDifferentTenant(); - doPost("/api/entityView", savedView, EntityView.class, status().isForbidden()); + + Mockito.reset(tbClusterService, auditLogService); + + doPost("/api/entityView", savedView) + .andExpect(status().isForbidden()) + .andExpect(statusReason(containsString(msgErrorPermission))); + + testNotifyEntityNever(savedView.getId(), savedView); + deleteDifferentTenant(); } @@ -174,20 +217,36 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes view.setCustomerId(customer.getId()); EntityView savedView = doPost("/api/entityView", view, EntityView.class); - doDelete("/api/entityView/" + savedView.getId().getId().toString()) + Mockito.reset(tbClusterService, auditLogService); + + String entityIdStr = savedView.getId().getId().toString(); + doDelete("/api/entityView/" + entityIdStr) .andExpect(status().isOk()); - doGet("/api/entityView/" + savedView.getId().getId().toString()) - .andExpect(status().isNotFound()); + testNotifyEntityBroadcastEntityStateChangeEventOneTimeMsgToEdgeServiceNever(savedView, savedView.getId(), savedView.getId(), + tenantId, view.getCustomerId(), tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.DELETED, entityIdStr); + + doGet("/api/entityView/" + entityIdStr) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Entity view",entityIdStr)))); } @Test public void testSaveEntityViewWithEmptyName() throws Exception { EntityView entityView = new EntityView(); entityView.setType("default"); + + Mockito.reset(tbClusterService, auditLogService); + + String msgError = "Entity view name " + msgErrorShouldBeSpecified; doPost("/api/entityView", entityView) .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Entity view name should be specified!"))); + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeServiceNeverError(entityView, + tenantId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -195,8 +254,16 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes EntityView view = getNewSavedEntityView("Test entity view"); Customer savedCustomer = doPost("/api/customer", getNewCustomer("My customer"), Customer.class); view.setCustomerId(savedCustomer.getId()); + + Mockito.reset(tbClusterService, auditLogService); + EntityView savedView = doPost("/api/entityView", view, EntityView.class); + testBroadcastEntityStateChangeEventTime(savedView.getId(), tenantId, 1); + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(savedView, savedView, + tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.UPDATED, ActionType.UPDATED, 1, 1, 5); + EntityView assignedView = doPost( "/api/customer/" + savedCustomer.getId().getId().toString() + "/entityView/" + savedView.getId().getId().toString(), EntityView.class); @@ -205,18 +272,38 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes EntityView foundView = doGet("/api/entityView/" + savedView.getId().getId().toString(), EntityView.class); assertEquals(savedCustomer.getId(), foundView.getCustomerId()); + testBroadcastEntityStateChangeEventNever(foundView.getId()); + testNotifyEntityAllOneTime(foundView, foundView.getId(), foundView.getId(), + tenantId, foundView.getCustomerId(), tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.ASSIGNED_TO_CUSTOMER, + foundView.getId().getId().toString(), foundView.getCustomerId().getId().toString(), savedCustomer.getTitle()); + EntityView unAssignedView = doDelete("/api/customer/entityView/" + savedView.getId().getId().toString(), EntityView.class); assertEquals(ModelConstants.NULL_UUID, unAssignedView.getCustomerId().getId()); foundView = doGet("/api/entityView/" + savedView.getId().getId().toString(), EntityView.class); assertEquals(ModelConstants.NULL_UUID, foundView.getCustomerId().getId()); + + testBroadcastEntityStateChangeEventNever(foundView.getId()); + testNotifyEntityAllOneTime(unAssignedView, savedView.getId(), savedView.getId(), + tenantId, savedView.getCustomerId(), tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.UNASSIGNED_FROM_CUSTOMER, + savedView.getCustomerId().getId().toString(), savedCustomer.getTitle()); } @Test public void testAssignEntityViewToNonExistentCustomer() throws Exception { EntityView savedView = getNewSavedEntityView("Test entity view"); - doPost("/api/customer/" + Uuids.timeBased().toString() + "/device/" + savedView.getId().getId().toString()) - .andExpect(status().isNotFound()); + + Mockito.reset(tbClusterService, auditLogService); + + String customerIdStr = Uuids.timeBased().toString(); + String msgError = msgErrorNoFound("Customer", customerIdStr); + doPost("/api/customer/" + customerIdStr + "/device/" + savedView.getId().getId().toString()) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityNever(savedView.getId(), savedView); } @Test @@ -242,8 +329,13 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes EntityView savedView = getNewSavedEntityView("Test entity view"); + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/customer/" + savedCustomer.getId().getId().toString() + "/entityView/" + savedView.getId().getId().toString()) - .andExpect(status().isForbidden()); + .andExpect(status().isForbidden()) + .andExpect(statusReason(containsString(msgErrorPermission))); + + testNotifyEntityNever(savedView.getId(), savedView); loginSysAdmin(); @@ -257,8 +349,11 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes CustomerId customerId = customer.getId(); String urlTemplate = "/api/customer/" + customerId.getId().toString() + "/entityViewInfos?"; - List> viewFutures = new ArrayList<>(128); - for (int i = 0; i < 128; i++) { + Mockito.reset(tbClusterService, auditLogService); + + int cntEntity = 128; + List> viewFutures = new ArrayList<>(cntEntity); + for (int i = 0; i < cntEntity; i++) { String entityName = "Test entity view " + i; viewFutures.add(executor.submit(() -> new EntityViewInfo(doPost("/api/customer/" + customerId.getId().toString() + "/entityView/" @@ -269,6 +364,15 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes List loadedViews = loadListOfInfo(new PageLink(23), urlTemplate); assertThat(entityViewInfos).containsExactlyInAnyOrderElementsOf(loadedViews); + + testNotifyEntityBroadcastEntityStateChangeEventMany(new EntityView(), new EntityView(), + tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.ADDED, ActionType.ADDED, cntEntity, 0, cntEntity*2, 0); + + testNotifyEntityBroadcastEntityStateChangeEventMany(new EntityView(), new EntityView(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.ASSIGNED_TO_CUSTOMER, ActionType.ASSIGNED_TO_CUSTOMER, cntEntity, cntEntity, + cntEntity*2, 3); } @Test @@ -289,12 +393,21 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes assertThat(namesOfView2).as(name2).containsExactlyInAnyOrderElementsOf(loadedNamesOfView2); deleteFutures.clear(); + + Mockito.reset(tbClusterService, auditLogService); + + int cntEntity = loadedNamesOfView1.size(); for (EntityView view : loadedNamesOfView1) { deleteFutures.add(executor.submit(() -> doDelete("/api/customer/entityView/" + view.getId().getId().toString()).andExpect(status().isOk()))); } Futures.allAsList(deleteFutures).get(TIMEOUT, SECONDS); + testBroadcastEntityStateChangeEventNever(loadedNamesOfView1.get(0).getId()); + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAnyAdditionalInfoAny(new EntityView(), new EntityView(), + tenantId, customerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, + ActionType.UNASSIGNED_FROM_CUSTOMER, ActionType.UNASSIGNED_FROM_CUSTOMER, cntEntity, cntEntity, 2); + PageData pageData = doGetTypedWithPageLink(urlTemplate, PAGE_DATA_ENTITY_VIEW_TYPE_REF, new PageLink(4, 0, name1)); Assert.assertFalse(pageData.hasNext()); @@ -370,10 +483,8 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes Set expectedActualAttributesSet = Set.of("caKey1", "caKey2", "caKey3", "caKey4"); Set actualAttributesSet = putAttributesAndWait("{\"caKey1\":\"value1\", \"caKey2\":true, \"caKey3\":42.0, \"caKey4\":73}", expectedActualAttributesSet); - log.debug("got correct actualAttributesSet, saving new entity view..."); EntityView savedView = getNewSavedEntityView("Test entity view"); - log.debug("fetching entity view telemetry..."); List> values = await("telemetry/ENTITY_VIEW") .atMost(TIMEOUT, SECONDS) .until(() -> doGetAsyncTyped("/api/plugins/telemetry/ENTITY_VIEW/" + savedView.getId().getId().toString() + @@ -381,7 +492,6 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes }), x -> x.size() >= expectedActualAttributesSet.size()); - log.debug("asserting..."); assertEquals("value1", getValue(values, "caKey1")); assertEquals(true, getValue(values, "caKey2")); assertEquals(42.0, getValue(values, "caKey3")); @@ -526,7 +636,6 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes getWsClient().subscribeLatestUpdate(keysToSubscribe, dtf); String viewDeviceId = testDevice.getId().getId().toString(); - log.debug("deviceid {}", viewDeviceId); DeviceCredentials deviceCredentials = doGet("/api/device/" + viewDeviceId + "/credentials", DeviceCredentials.class); assertEquals(testDevice.getId(), deviceCredentials.getDeviceId()); @@ -534,7 +643,6 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes String accessToken = deviceCredentials.getCredentialsId(); assertNotNull(accessToken); - log.debug("creating mqtt client..."); String clientId = MqttAsyncClient.generateClientId(); MqttAsyncClient client = new MqttAsyncClient("tcp://localhost:1883", clientId, new MemoryPersistence()); @@ -542,14 +650,11 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes options.setUserName(accessToken); client.connect(options); awaitConnected(client, SECONDS.toMillis(30)); - log.debug("mqtt connected..."); MqttMessage message = new MqttMessage(); message.setPayload((stringKV).getBytes()); getWsClient().registerWaitForUpdate(); IMqttDeliveryToken token = client.publish("v1/devices/me/attributes", message); - log.debug("publish token.message {}", token.getMessage()); await("mqtt ack").pollInterval(5, MILLISECONDS).atMost(TIMEOUT, SECONDS).until(() -> token.getMessage() == null); - log.debug("token.message delivered {}", token.getMessage()); assertThat(getWsClient().waitForUpdate()).as("ws update received").isNotBlank(); return getAttributeKeys("DEVICE", viewDeviceId); } From 35275d73c3c9076253847a2755f5df65e70b0683 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 5 Jul 2022 13:45:44 +0300 Subject: [PATCH 04/28] refactoring: tests EntityView: save ADD change --- .../server/controller/BaseEntityViewControllerTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 84e617f3fa..4f8737270e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -145,9 +145,10 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes assertEquals(savedView, foundEntityView); - testNotifyEntityBroadcastEntityStateChangeEventOneTimeMsgToEdgeServiceNever(foundEntityView, foundEntityView.getId(), foundEntityView.getId(), + testBroadcastEntityStateChangeEventTime(foundEntityView.getId(), tenantId, 1); + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundEntityView, foundEntityView, tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, - ActionType.ADDED); + ActionType.ADDED, ActionType.ADDED, 1, 0, 1); savedView.setName("New test entity view"); From fc0fae6c83054dc277c23b58074f73ea67df240f Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Tue, 5 Jul 2022 23:08:46 +0300 Subject: [PATCH 05/28] refactoring: tests EntityView: License Headers --- .../server/controller/AbstractNotifyEntityTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index b5c26e006c..60af48b8d8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 1e1f8edb1f0124705feb8a8729069cd067343045 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Wed, 6 Jul 2022 13:40:13 +0300 Subject: [PATCH 06/28] refactoring: tests EntityOtaPackageController --- .../controller/AbstractNotifyEntityTest.java | 6 + .../BaseOtaPackageControllerTest.java | 105 +++++++++++++++--- 2 files changed, 93 insertions(+), 18 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 60af48b8d8..85a2fbc9e6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -587,6 +587,12 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } private String entityClassToEntityTypeName(HasName entity) { + String entityType = entityClassToString(entity); + return "SAVE_OTA_PACKAGE_INFO_REQUEST".equals(entityType) || "OTA_PACKAGE_INFO".equals(entityType)? + EntityType.OTA_PACKAGE.name().toUpperCase(Locale.ENGLISH) : entityType; + } + + private String entityClassToString(HasName entity) { String className = entity.getClass().toString() .substring(entity.getClass().toString().lastIndexOf(".") + 1); List str = className.chars() diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java index cbc00b0987..8557c7c008 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java @@ -21,6 +21,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.mock.web.MockMultipartFile; import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; @@ -31,10 +32,12 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.exception.DataValidationException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -102,6 +105,8 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes firmwareInfo.setVersion(VERSION); firmwareInfo.setUsesUrl(false); + Mockito.reset(tbClusterService, auditLogService); + OtaPackageInfo savedFirmwareInfo = save(firmwareInfo); Assert.assertNotNull(savedFirmwareInfo); @@ -111,12 +116,20 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes Assert.assertEquals(firmwareInfo.getTitle(), savedFirmwareInfo.getTitle()); Assert.assertEquals(firmwareInfo.getVersion(), savedFirmwareInfo.getVersion()); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedFirmwareInfo, savedFirmwareInfo.getId(), savedFirmwareInfo.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED); + savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode()); save(new SaveOtaPackageInfoRequest(savedFirmwareInfo, false)); OtaPackageInfo foundFirmwareInfo = doGet("/api/otaPackage/info/" + savedFirmwareInfo.getId().getId().toString(), OtaPackageInfo.class); Assert.assertEquals(foundFirmwareInfo.getTitle(), savedFirmwareInfo.getTitle()); + + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundFirmwareInfo, foundFirmwareInfo.getId(), foundFirmwareInfo.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UPDATED); } @Test @@ -127,14 +140,42 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes firmwareInfo.setTitle(RandomStringUtils.randomAlphabetic(300)); firmwareInfo.setVersion(VERSION); firmwareInfo.setUsesUrl(false); - doPost("/api/otaPackage", firmwareInfo).andExpect(statusReason(containsString("length of title must be equal or less than 255"))); + String msgError = msgErrorFieldLength("title"); + + Mockito.reset(tbClusterService, auditLogService); + + doPost("/api/otaPackage", firmwareInfo) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgError))); + + firmwareInfo.setTenantId(savedTenant.getId()); + testNotifyEntityEqualsOneTimeServiceNeverError(firmwareInfo, + savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, new DataValidationException(msgError)); + firmwareInfo.setTitle(TITLE); firmwareInfo.setVersion(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/otaPackage", firmwareInfo).andExpect(statusReason(containsString("length of version must be equal or less than 255"))); + msgError = msgErrorFieldLength("version"); + doPost("/api/otaPackage", firmwareInfo) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgError))); + + firmwareInfo.setTenantId(savedTenant.getId()); + testNotifyEntityEqualsOneTimeServiceNeverError(firmwareInfo, + savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, new DataValidationException(msgError)); + firmwareInfo.setVersion(VERSION); firmwareInfo.setUsesUrl(true); + msgError = msgErrorFieldLength("url"); firmwareInfo.setUrl(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/otaPackage", firmwareInfo).andExpect(statusReason(containsString("length of url must be equal or less than 255"))); + doPost("/api/otaPackage", firmwareInfo) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgError))); + + firmwareInfo.setTenantId(savedTenant.getId()); + testNotifyEntityEqualsOneTimeServiceNeverError(firmwareInfo, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -164,12 +205,19 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + Mockito.reset(tbClusterService, auditLogService); + + OtaPackage savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); Assert.assertEquals(FILE_NAME, savedFirmware.getFileName()); Assert.assertEquals(CONTENT_TYPE, savedFirmware.getContentType()); Assert.assertEquals(CHECKSUM_ALGORITHM, savedFirmware.getChecksumAlgorithm().name()); Assert.assertEquals(CHECKSUM, savedFirmware.getChecksum()); + + testNotifyEntityAllOneTime(savedFirmware, savedFirmware.getId(), savedFirmware.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UPDATED); } @Test @@ -184,10 +232,16 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes OtaPackageInfo savedFirmwareInfo = save(firmwareInfo); loginDifferentTenant(); + + Mockito.reset(tbClusterService, auditLogService); + doPost("/api/otaPackage", - new SaveOtaPackageInfoRequest(savedFirmwareInfo, false), - OtaPackageInfo.class, - status().isForbidden()); + new SaveOtaPackageInfoRequest(savedFirmwareInfo, false)) + .andExpect(status().isForbidden()) + .andExpect(statusReason(containsString(msgErrorPermission))); + + testNotifyEntityNever(savedFirmwareInfo.getId(), savedFirmwareInfo); + deleteDifferentTenant(); } @@ -220,11 +274,12 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); OtaPackage foundFirmware = doGet("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString(), OtaPackage.class); Assert.assertNotNull(foundFirmware); - Assert.assertEquals(savedFirmware, new OtaPackageInfo(foundFirmware)); + Assert.assertEquals(savedFirmware, foundFirmware); Assert.assertEquals(DATA, foundFirmware.getData()); } @@ -239,17 +294,28 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes OtaPackageInfo savedFirmwareInfo = save(firmwareInfo); + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString()) .andExpect(status().isOk()); + testNotifyEntityAllOneTime(savedFirmwareInfo, savedFirmwareInfo.getId(), savedFirmwareInfo.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedFirmwareInfo.getId().getId().toString()); + doGet("/api/otaPackage/info/" + savedFirmwareInfo.getId().getId().toString()) .andExpect(status().isNotFound()); } @Test public void testFindTenantFirmwares() throws Exception { + + Mockito.reset(tbClusterService, auditLogService); + List otaPackages = new ArrayList<>(); - for (int i = 0; i < 165; i++) { + int cntEntity = 165; + int startIndexSaveData = 101; + for (int i = 0; i < cntEntity; i++) { SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); firmwareInfo.setDeviceProfileId(deviceProfileId); firmwareInfo.setType(FIRMWARE); @@ -259,16 +325,19 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes OtaPackageInfo savedFirmwareInfo = save(firmwareInfo); - if (i > 100) { + if (i >= startIndexSaveData) { MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); - otaPackages.add(savedFirmware); - } else { - otaPackages.add(savedFirmwareInfo); + OtaPackage savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + savedFirmwareInfo = new OtaPackageInfo(savedFirmware); } + otaPackages.add(savedFirmwareInfo); } + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new OtaPackageInfo(), new OtaPackageInfo(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, ActionType.ADDED, cntEntity, 0, (cntEntity*2 - startIndexSaveData)); + List loadedFirmwares = new ArrayList<>(); PageLink pageLink = new PageLink(24); PageData pageData; @@ -306,7 +375,7 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes if (i > 100) { MockMultipartFile testData = new MockMultipartFile("file", FILE_NAME, CONTENT_TYPE, DATA.array()); - OtaPackageInfo savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); + OtaPackage savedFirmware = savaData("/api/otaPackage/" + savedFirmwareInfo.getId().getId().toString() + "?checksum={checksum}&checksumAlgorithm={checksumAlgorithm}", testData, CHECKSUM, CHECKSUM_ALGORITHM); savedFirmwareInfo = new OtaPackageInfo(savedFirmware); otaPackagesWithData.add(savedFirmwareInfo); } @@ -352,11 +421,11 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes return doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); } - protected OtaPackageInfo savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception { + protected OtaPackage savaData(String urlTemplate, MockMultipartFile content, String... params) throws Exception { MockMultipartHttpServletRequestBuilder postRequest = MockMvcRequestBuilders.multipart(urlTemplate, params); postRequest.file(content); setJwtToken(postRequest); - return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackageInfo.class); + return readResponse(mockMvc.perform(postRequest).andExpect(status().isOk()), OtaPackage.class); } } From f2dd9045227ba9c97d5ab6727c9ab23d42afe668 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Wed, 6 Jul 2022 14:03:33 +0300 Subject: [PATCH 07/28] refactoring: tests add sgErrorNotFound --- .../server/controller/AbstractNotifyEntityTest.java | 1 + .../server/controller/BaseEntityRelationControllerTest.java | 4 ++-- .../server/controller/BaseEntityViewControllerTest.java | 2 +- .../server/controller/BaseOtaPackageControllerTest.java | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 85a2fbc9e6..51b26649c0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -63,6 +63,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { protected final String msgErrorPermission = "You don't have permission to perform this operation!"; protected final String msgErrorShouldBeSpecified = "should be specified"; + protected final String msgErrorNotFound = "Requested item wasn't found!"; protected void testNotifyEntityAllOneTime(HasName entity, EntityId entityId, EntityId originatorId, diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java index f78fd6ce8b..2d28e32906 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java @@ -354,7 +354,7 @@ public abstract class BaseEntityRelationControllerTest extends AbstractControlle doDelete(url) .andExpect(status().isNotFound()) - .andExpect(statusReason(containsString("Requested item wasn't found!"))); + .andExpect(statusReason(containsString(msgErrorNotFound))); testNotifyEntityNever(mainDevice.getId(), null); } @@ -376,7 +376,7 @@ public abstract class BaseEntityRelationControllerTest extends AbstractControlle doDelete(url) .andExpect(status().isNotFound()) - .andExpect(statusReason(containsString("Requested item wasn't found!"))); + .andExpect(statusReason(containsString(msgErrorNotFound))); testNotifyEntityNever(mainDevice.getId(), null); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 4f8737270e..55ec008d56 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -164,7 +164,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes doGet("/api/tenant/entityViews?entityViewName=" + name) .andExpect(status().isNotFound()) - .andExpect(statusReason(containsString("Requested item wasn't found!"))); + .andExpect(statusReason(containsString(msgErrorNotFound))); } @Test diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java index 8557c7c008..18d76d96e8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseOtaPackageControllerTest.java @@ -304,7 +304,8 @@ public abstract class BaseOtaPackageControllerTest extends AbstractControllerTes ActionType.DELETED, savedFirmwareInfo.getId().getId().toString()); doGet("/api/otaPackage/info/" + savedFirmwareInfo.getId().getId().toString()) - .andExpect(status().isNotFound()); + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNotFound))); } @Test From de74ded2d24f4615e8fab3cdf869075f4f12497c Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Wed, 6 Jul 2022 16:57:27 +0300 Subject: [PATCH 08/28] refactoring: tests RuleChain --- .../BaseRuleChainControllerTest.java | 64 +++++++++++++++++-- 1 file changed, 57 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java index 9395f99366..2b6784dfa6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java @@ -21,14 +21,17 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.Collections; @@ -75,22 +78,45 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest public void testSaveRuleChain() throws Exception { RuleChain ruleChain = new RuleChain(); ruleChain.setName("RuleChain"); + + Mockito.reset(tbClusterService, auditLogService); + RuleChain savedRuleChain = doPost("/api/ruleChain", ruleChain, RuleChain.class); Assert.assertNotNull(savedRuleChain); Assert.assertNotNull(savedRuleChain.getId()); Assert.assertTrue(savedRuleChain.getCreatedTime() > 0); Assert.assertEquals(ruleChain.getName(), savedRuleChain.getName()); + + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedRuleChain, savedRuleChain.getId(), savedRuleChain.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED); + savedRuleChain.setName("New RuleChain"); doPost("/api/ruleChain", savedRuleChain, RuleChain.class); RuleChain foundRuleChain = doGet("/api/ruleChain/" + savedRuleChain.getId().getId().toString(), RuleChain.class); Assert.assertEquals(savedRuleChain.getName(), foundRuleChain.getName()); + + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedRuleChain, savedRuleChain.getId(), savedRuleChain.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UPDATED); } @Test public void testSaveRuleChainWithViolationOfLengthValidation() throws Exception { + + Mockito.reset(tbClusterService, auditLogService); + RuleChain ruleChain = new RuleChain(); ruleChain.setName(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/ruleChain", ruleChain).andExpect(statusReason(containsString("length of name must be equal or less than 255"))); + String msgError = msgErrorFieldLength("name"); + doPost("/api/ruleChain", ruleChain) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgError))); + + ruleChain.setTenantId(savedTenant.getId()); + testNotifyEntityEqualsOneTimeServiceNeverError(ruleChain, + savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -109,11 +135,19 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest ruleChain.setName("RuleChain"); RuleChain savedRuleChain = doPost("/api/ruleChain", ruleChain, RuleChain.class); + Mockito.reset(tbClusterService, auditLogService); + + String entityIdStr = savedRuleChain.getId().getId().toString(); doDelete("/api/ruleChain/" + savedRuleChain.getId().getId().toString()) .andExpect(status().isOk()); - doGet("/api/ruleChain/" + savedRuleChain.getId().getId().toString()) - .andExpect(status().isNotFound()); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedRuleChain, savedRuleChain.getId(), savedRuleChain.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedRuleChain.getId().getId().toString()); + + doGet("/api/ruleChain/" + entityIdStr) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Rule chain", entityIdStr)))); } @Test @@ -124,12 +158,16 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest List edgeRuleChains = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId() + "/ruleChains?", - new TypeReference<>() {}, pageLink); + new TypeReference<>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); edgeRuleChains.addAll(pageData.getData()); - for (int i = 0; i < 28; i++) { + Mockito.reset(tbClusterService, auditLogService); + + int cntEntity = 28; + for (int i = 0; i < cntEntity; i++) { RuleChain ruleChain = new RuleChain(); ruleChain.setName("RuleChain " + i); ruleChain.setType(RuleChainType.EDGE); @@ -139,11 +177,16 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest edgeRuleChains.add(savedRuleChain); } + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new RuleChain(), new RuleChain(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, ActionType.ADDED, cntEntity, 0, cntEntity * 2); + List loadedEdgeRuleChains = new ArrayList<>(); pageLink = new PageLink(17); do { pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId() + "/ruleChains?", - new TypeReference<>() {}, pageLink); + new TypeReference<>() { + }, pageLink); loadedEdgeRuleChains.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); @@ -155,6 +198,8 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest Assert.assertEquals(edgeRuleChains, loadedEdgeRuleChains); + Mockito.reset(tbClusterService, auditLogService); + for (RuleChain ruleChain : loadedEdgeRuleChains) { if (!ruleChain.isRoot()) { doDelete("/api/edge/" + savedEdge.getId().getId().toString() @@ -162,9 +207,14 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest } } + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAnyAdditionalInfoAny(new RuleChain(), new RuleChain(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UNASSIGNED_FROM_EDGE, ActionType.UNASSIGNED_FROM_EDGE, cntEntity, cntEntity, 3); + pageLink = new PageLink(17); pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId() + "/ruleChains?", - new TypeReference<>() {}, pageLink); + new TypeReference<>() { + }, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); } From 2301a918a7691300c20837f34334161e4b9282b6 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Wed, 6 Jul 2022 19:30:46 +0300 Subject: [PATCH 09/28] refactoring: tests RuleChain add with ActionType.ASSIGNED_TO_EDGE --- .../server/controller/AbstractNotifyEntityTest.java | 1 - .../server/controller/BaseDeviceControllerTest.java | 1 + .../server/controller/BaseDeviceProfileControllerTest.java | 1 + .../server/controller/BaseEntityViewControllerTest.java | 2 ++ .../server/controller/BaseRuleChainControllerTest.java | 7 ++++++- 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 51b26649c0..1eff1a2513 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -169,7 +169,6 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, extractMatcherAdditionalInfoClass(additionalInfo)); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTimeRuleEngine); - Mockito.reset(tbClusterService, auditLogService); } protected void testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAnyAdditionalInfoAny(HasName entity, HasName originator, diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index 273ec4f82f..d6a77a1d6b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -813,6 +813,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new Device(), new Device(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, ActionType.ASSIGNED_TO_CUSTOMER, cntEntity, cntEntity, cntEntity * 2); + Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); testNotificationUpdateGatewayNever(); List loadedDevices = new ArrayList<>(cntEntity); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 75f40bae8e..61d35a972b 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -387,6 +387,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new DeviceProfile(), new DeviceProfile(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, ActionType.ADDED, cntEntity, cntEntity, cntEntity); + Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); List loadedDeviceProfiles = new ArrayList<>(); pageLink = new PageLink(17); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index 55ec008d56..a66f3e4095 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -149,6 +149,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundEntityView, foundEntityView, tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED, ActionType.ADDED, 1, 0, 1); + Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); savedView.setName("New test entity view"); @@ -264,6 +265,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(savedView, savedView, tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.UPDATED, ActionType.UPDATED, 1, 1, 5); + Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); EntityView assignedView = doPost( "/api/customer/" + savedCustomer.getId().getId().toString() + "/entityView/" + savedView.getId().getId().toString(), diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java index 2b6784dfa6..62880848e9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java @@ -141,7 +141,7 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest doDelete("/api/ruleChain/" + savedRuleChain.getId().getId().toString()) .andExpect(status().isOk()); - testNotifyEntityOneTimeMsgToEdgeServiceNever(savedRuleChain, savedRuleChain.getId(), savedRuleChain.getId(), + testNotifyEntityBroadcastEntityStateChangeEventOneTimeMsgToEdgeServiceNever(savedRuleChain, savedRuleChain.getId(), savedRuleChain.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.DELETED, savedRuleChain.getId().getId().toString()); @@ -155,6 +155,7 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest Edge edge = constructEdge("My edge", "default"); Edge savedEdge = doPost("/api/edge", edge, Edge.class); + List edgeRuleChains = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData pageData = doGetTypedWithPageLink("/api/edge/" + savedEdge.getId().getId() + "/ruleChains?", @@ -180,6 +181,10 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new RuleChain(), new RuleChain(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, ActionType.ADDED, cntEntity, 0, cntEntity * 2); + testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new RuleChain(), new RuleChain(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ASSIGNED_TO_EDGE, ActionType.ASSIGNED_TO_EDGE, cntEntity, cntEntity, cntEntity * 2, + new String(), new String(), new String()); List loadedEdgeRuleChains = new ArrayList<>(); pageLink = new PageLink(17); From d0d47f96dbb8589a9be6770161e15e866e5cc8dc Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Thu, 7 Jul 2022 04:40:47 +0300 Subject: [PATCH 10/28] refactoring: tests fix bug testSaveEntityView --- .../server/controller/BaseDeviceControllerTest.java | 3 +-- .../server/controller/BaseDeviceProfileControllerTest.java | 2 +- .../server/controller/BaseEntityViewControllerTest.java | 4 ++-- .../server/controller/BaseRuleChainControllerTest.java | 3 +-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index d6a77a1d6b..4c4f44aea6 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -815,6 +815,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { ActionType.ADDED, ActionType.ASSIGNED_TO_CUSTOMER, cntEntity, cntEntity, cntEntity * 2); Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); testNotificationUpdateGatewayNever(); + Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); List loadedDevices = new ArrayList<>(cntEntity); PageLink pageLink = new PageLink(23); @@ -829,8 +830,6 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { assertThat(devices).containsExactlyInAnyOrderElementsOf(loadedDevices); - Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); - deleteEntitiesAsync("/api/customer/device/", loadedDevices, executor).get(TIMEOUT, TimeUnit.SECONDS); testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAnyAdditionalInfoAny(new Device(), new Device(), diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 61d35a972b..1fb5316a17 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -387,7 +387,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(new DeviceProfile(), new DeviceProfile(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, ActionType.ADDED, cntEntity, cntEntity, cntEntity); - Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); + Mockito.reset(tbClusterService, auditLogService); List loadedDeviceProfiles = new ArrayList<>(); pageLink = new PageLink(17); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java index a66f3e4095..f1121dfb14 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseEntityViewControllerTest.java @@ -149,7 +149,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(foundEntityView, foundEntityView, tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.ADDED, ActionType.ADDED, 1, 0, 1); - Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); + Mockito.reset(tbClusterService, auditLogService); savedView.setName("New test entity view"); @@ -265,7 +265,7 @@ public abstract class BaseEntityViewControllerTest extends AbstractControllerTes testNotifyManyEntityManyTimeMsgToEdgeServiceEntityEqAny(savedView, savedView, tenantId, tenantAdminCustomerId, tenantAdminUserId, TENANT_ADMIN_EMAIL, ActionType.UPDATED, ActionType.UPDATED, 1, 1, 5); - Mockito.reset(tbClusterService, auditLogService, gatewayNotificationsService); + Mockito.reset(tbClusterService, auditLogService); EntityView assignedView = doPost( "/api/customer/" + savedCustomer.getId().getId().toString() + "/entityView/" + savedView.getId().getId().toString(), diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java index 62880848e9..33602bbfa9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseRuleChainControllerTest.java @@ -185,6 +185,7 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ASSIGNED_TO_EDGE, ActionType.ASSIGNED_TO_EDGE, cntEntity, cntEntity, cntEntity * 2, new String(), new String(), new String()); + Mockito.reset(tbClusterService, auditLogService); List loadedEdgeRuleChains = new ArrayList<>(); pageLink = new PageLink(17); @@ -203,8 +204,6 @@ public abstract class BaseRuleChainControllerTest extends AbstractControllerTest Assert.assertEquals(edgeRuleChains, loadedEdgeRuleChains); - Mockito.reset(tbClusterService, auditLogService); - for (RuleChain ruleChain : loadedEdgeRuleChains) { if (!ruleChain.isRoot()) { doDelete("/api/edge/" + savedEdge.getId().getId().toString() From d423738a4890f041d5f7aff00de3166918e16819 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Thu, 7 Jul 2022 15:15:27 +0300 Subject: [PATCH 11/28] refactoring: tests TbResource --- .../controller/AbstractNotifyEntityTest.java | 98 +++++++++++++------ .../BaseDashboardControllerTest.java | 1 + .../BaseTbResourceControllerTest.java | 72 ++++++++++++-- 3 files changed, 134 insertions(+), 37 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 1eff1a2513..a32ec3c427 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -86,11 +86,15 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.eq(edgeTypeByActionType(actionType))); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(relation.getTo()); ArgumentMatcher matcherEntityClassEquals = Objects::isNull; - testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, extractMatcherAdditionalInfo(additionalInfo)); testPushMsgToRuleEngineNever(relation.getTo()); matcherOriginatorId = argument -> argument.equals(relation.getFrom()); - testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, extractMatcherAdditionalInfo(additionalInfo)); testPushMsgToRuleEngineNever(relation.getFrom()); Mockito.reset(tbClusterService, auditLogService); @@ -104,7 +108,11 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { Mockito.eq(edgeTypeByActionType(actionType))); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(relation.getFrom().getClass()); ArgumentMatcher matcherEntityClassEquals = Objects::isNull; - testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime * 2, 1); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); Mockito.reset(tbClusterService, auditLogService); @@ -152,7 +160,11 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { testNotificationMsgToEdgeServiceNeverWithActionType(entityId, actionType); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); - testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, extractMatcherAdditionalInfo(additionalInfo)); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); Mockito.reset(tbClusterService, auditLogService); @@ -166,7 +178,11 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { testSendNotificationMsgToEdgeServiceTimeEntityEqAny(tenantId, actionTypeEdge, cntTimeEdge); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); - testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfo(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, extractMatcherAdditionalInfoClass(additionalInfo)); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTimeRuleEngine); } @@ -178,7 +194,11 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { testSendNotificationMsgToEdgeServiceTimeEntityEqAny(tenantId, actionTypeEdge, cntTimeEdge); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); - testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, cntAdditionalInfo); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTimeEdge); Mockito.reset(tbClusterService, auditLogService); @@ -192,7 +212,11 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { testNotificationMsgToEdgeServiceNeverWithActionType(entityId, actionType); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); - testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, cntAdditionalInfo); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTime); Mockito.reset(tbClusterService, auditLogService); @@ -233,7 +257,11 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { testNotificationMsgToEdgeServiceTime(entityId, tenantId, actionTypeEdge, cntTimeEdge); ArgumentMatcher matcherEntityClassEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.getClass().equals(originatorId.getClass()); - testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, customerId, userId, userName, actionType, cntTime, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfoAny(matcherEntityClassEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, cntAdditionalInfo); testPushMsgToRuleEngineTime(matcherOriginatorId, tenantId, cntTimeRuleEngine); testBroadcastEntityStateChangeEventTime(entityId, tenantId, cntTime); @@ -364,9 +392,13 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { private void testLogEntityAction(HasName entity, EntityId originatorId, TenantId tenantId, CustomerId customerId, UserId userId, String userName, ActionType actionType, int cntTime, Object... additionalInfo) { - ArgumentMatcher matcherEntityEquals = entity == null ? Objects::isNull : argument -> argument.equals(entity); + ArgumentMatcher matcherEntityEquals = entity == null ? Objects::isNull : argument -> argument.toString().equals(entity.toString()); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); - testLogEntityActionAdditionalInfo(matcherEntityEquals, matcherOriginatorId, tenantId, customerId, userId, userName, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfo(matcherEntityEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, extractMatcherAdditionalInfo(additionalInfo)); } @@ -375,19 +407,24 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { ActionType actionType, int cntTime, Object... additionalInfo) { ArgumentMatcher matcherEntityEquals = argument -> argument.getClass().equals(entity.getClass()); ArgumentMatcher matcherOriginatorId = argument -> argument.equals(originatorId); - testLogEntityActionAdditionalInfo(matcherEntityEquals, matcherOriginatorId, tenantId, customerId, userId, userName, + ArgumentMatcher matcherCustomerId = customerId == null ? + argument -> argument.getClass().equals(CustomerId.class) : argument -> argument.equals(customerId); + ArgumentMatcher matcherUserId = userId == null ? + argument -> argument.getClass().equals(UserId.class) : argument -> argument.equals(userId); + testLogEntityActionAdditionalInfo(matcherEntityEquals, matcherOriginatorId, tenantId, matcherCustomerId, matcherUserId, userName, actionType, cntTime, extractMatcherAdditionalInfo(additionalInfo)); } private void testLogEntityActionAdditionalInfo(ArgumentMatcher matcherEntity, ArgumentMatcher matcherOriginatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, - ActionType actionType, int cntTime, List> matcherAdditionalInfos) { + TenantId tenantId, ArgumentMatcher matcherCustomerId, + ArgumentMatcher matcherUserId, String userName, ActionType actionType, + int cntTime, List> matcherAdditionalInfos) { switch (matcherAdditionalInfos.size()) { case 1: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), @@ -398,8 +435,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { case 2: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), @@ -411,8 +448,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { case 3: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), @@ -425,8 +462,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { default: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), @@ -436,14 +473,15 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } private void testLogEntityActionAdditionalInfoAny(ArgumentMatcher matcherEntity, ArgumentMatcher matcherOriginatorId, - TenantId tenantId, CustomerId customerId, UserId userId, String userName, + TenantId tenantId, ArgumentMatcher matcherCustomerId, + ArgumentMatcher matcherUserId, String userName, ActionType actionType, int cntTime, int cntAdditionalInfo) { switch (cntAdditionalInfo) { case 1: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), @@ -454,8 +492,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { case 2: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), @@ -467,8 +505,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { case 3: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), @@ -481,8 +519,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { default: Mockito.verify(auditLogService, times(cntTime)) .logEntityAction(Mockito.eq(tenantId), - Mockito.eq(customerId), - Mockito.eq(userId), + Mockito.argThat(matcherCustomerId), + Mockito.argThat(matcherUserId), Mockito.eq(userName), Mockito.argThat(matcherOriginatorId), Mockito.argThat(matcherEntity), diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDashboardControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDashboardControllerTest.java index cfb9489f69..66fc1a5178 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDashboardControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDashboardControllerTest.java @@ -283,6 +283,7 @@ public abstract class BaseDashboardControllerTest extends AbstractControllerTest doDelete("/api/tenant/" + savedTenant2.getId().getId().toString()) .andExpect(status().isForbidden()) .andExpect(statusReason(containsString(msgErrorPermission))); + testNotifyEntityNever(savedDashboard.getId(), savedDashboard); loginSysAdmin(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java index 4088e78563..36d8d56fbc 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java @@ -21,14 +21,17 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; import java.util.Collections; @@ -75,6 +78,9 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes @Test public void testSaveTbResource() throws Exception { + + Mockito.reset(tbClusterService, auditLogService); + TbResource resource = new TbResource(); resource.setResourceType(ResourceType.JKS); resource.setTitle("My first resource"); @@ -83,6 +89,10 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes TbResource savedResource = save(resource); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedResource, savedResource.getId(), savedResource.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED); + Assert.assertNotNull(savedResource); Assert.assertNotNull(savedResource.getId()); Assert.assertTrue(savedResource.getCreatedTime() > 0); @@ -98,6 +108,10 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes TbResource foundResource = doGet("/api/resource/" + savedResource.getId().getId().toString(), TbResource.class); Assert.assertEquals(foundResource.getTitle(), savedResource.getTitle()); + + testNotifyEntityOneTimeMsgToEdgeServiceNever(foundResource, foundResource.getId(), foundResource.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.UPDATED); } @Test @@ -107,7 +121,16 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes resource.setTitle(RandomStringUtils.randomAlphabetic(300)); resource.setFileName(DEFAULT_FILE_NAME); resource.setData("Test Data"); - doPost("/api/resource", resource).andExpect(statusReason(containsString("length of title must be equal or less than 255"))); + + Mockito.reset(tbClusterService, auditLogService); + + String msgError = msgErrorFieldLength("title"); + doPost("/api/resource", resource) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgError))); + + testNotifyEntityEqualsOneTimeServiceNeverError(resource, savedTenant.getId(), + tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(msgError)); } @Test @@ -118,10 +141,24 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes resource.setFileName(DEFAULT_FILE_NAME); resource.setData("Test Data"); - TbResource savedResource = save(resource); + TbResource savedResource = save(resource); loginDifferentTenant(); - doPostWithTypedResponse("/api/resource", savedResource, new TypeReference<>(){}, status().isForbidden()); + + Mockito.reset(tbClusterService, auditLogService); + + doPost("/api/resource", savedResource) + .andExpect(status().isForbidden()) + .andExpect(statusReason(containsString(msgErrorPermission))); + + testNotifyEntityNever(savedResource.getId(), savedResource); + + doDelete("/api/resource/" + savedResource.getId().getId().toString()) + .andExpect(status().isForbidden()) + .andExpect(statusReason(containsString(msgErrorPermission))); + + testNotifyEntityNever(savedResource.getId(), savedResource); + deleteDifferentTenant(); } @@ -150,17 +187,27 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes TbResource savedResource = save(resource); + Mockito.reset(tbClusterService, auditLogService); + doDelete("/api/resource/" + savedResource.getId().getId().toString()) .andExpect(status().isOk()); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedResource, savedResource.getId(), savedResource.getId(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.DELETED, savedResource.getId().getId().toString()); + doGet("/api/resource/" + savedResource.getId().getId().toString()) .andExpect(status().isNotFound()); } @Test public void testFindTenantTbResources() throws Exception { + + Mockito.reset(tbClusterService, auditLogService); + List resources = new ArrayList<>(); - for (int i = 0; i < 173; i++) { + int cntEntity = 173; + for (int i = 0; i < cntEntity; i++) { TbResource resource = new TbResource(); resource.setTitle("Resource" + i); resource.setResourceType(ResourceType.JKS); @@ -173,7 +220,7 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes PageData pageData; do { pageData = doGetTypedWithPageLink("/api/resource?", - new TypeReference>() { + new TypeReference<>() { }, pageLink); loadedResources.addAll(pageData.getData()); if (pageData.hasNext()) { @@ -181,6 +228,10 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes } } while (pageData.hasNext()); + testNotifyManyEntityManyTimeMsgToEdgeServiceNever(new TbResource(), new TbResource(), + savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), + ActionType.ADDED, cntEntity); + Collections.sort(resources, idComparator); Collections.sort(loadedResources, idComparator); @@ -205,7 +256,7 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes PageData pageData; do { pageData = doGetTypedWithPageLink("/api/resource?", - new TypeReference>() { + new TypeReference<>() { }, pageLink); loadedResources.addAll(pageData.getData()); if (pageData.hasNext()) { @@ -218,16 +269,23 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes Assert.assertEquals(resources, loadedResources); + Mockito.reset(tbClusterService, auditLogService); + + int cntEntity = resources.size(); for (TbResourceInfo resource : resources) { doDelete("/api/resource/" + resource.getId().getId().toString()) .andExpect(status().isOk()); } + testNotifyManyEntityManyTimeMsgToEdgeServiceNeverAdditionalInfoAny(new TbResource(), new TbResource(), + resources.get(0).getTenantId(), null, null, SYS_ADMIN_EMAIL, + ActionType.DELETED, cntEntity, 1); + pageLink = new PageLink(27); loadedResources.clear(); do { pageData = doGetTypedWithPageLink("/api/resource?", - new TypeReference>() { + new TypeReference<>() { }, pageLink); loadedResources.addAll(pageData.getData()); if (pageData.hasNext()) { From c8a8b4630bd64272d44c153e9964edc291c60540 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Thu, 7 Jul 2022 15:36:16 +0300 Subject: [PATCH 12/28] refactoring: tests TbResource add msgNotFound --- .../controller/BaseTbResourceControllerTest.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java index 36d8d56fbc..d3153f57bc 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTbResourceControllerTest.java @@ -188,16 +188,18 @@ public abstract class BaseTbResourceControllerTest extends AbstractControllerTes TbResource savedResource = save(resource); Mockito.reset(tbClusterService, auditLogService); - - doDelete("/api/resource/" + savedResource.getId().getId().toString()) + String resourceIdStr = savedResource.getId().getId().toString(); + doDelete("/api/resource/" + resourceIdStr) .andExpect(status().isOk()); + testNotifyEntityOneTimeMsgToEdgeServiceNever(savedResource, savedResource.getId(), savedResource.getId(), savedTenant.getId(), tenantAdmin.getCustomerId(), tenantAdmin.getId(), tenantAdmin.getEmail(), - ActionType.DELETED, savedResource.getId().getId().toString()); + ActionType.DELETED, resourceIdStr); doGet("/api/resource/" + savedResource.getId().getId().toString()) - .andExpect(status().isNotFound()); + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Resource", resourceIdStr)))); } @Test From bbc43c2572bb9aa44dc939969fc9486567dc7dbc Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 7 Jul 2022 17:15:59 +0300 Subject: [PATCH 13/28] Graceful shutdown JavaScript Executor Microservice and minor change in logs --- msa/js-executor/api/httpServer.ts | 31 ++++++++++--- msa/js-executor/queue/awsSqsTemplate.ts | 22 +++++---- msa/js-executor/queue/kafkaTemplate.ts | 27 ++++++----- msa/js-executor/queue/pubSubTemplate.ts | 19 ++++---- msa/js-executor/queue/queue.models.ts | 2 +- msa/js-executor/queue/rabbitmqTemplate.ts | 29 ++++++------ msa/js-executor/queue/serviceBusTemplate.ts | 33 +++++++++----- msa/js-executor/server.ts | 50 ++++++++++++--------- 8 files changed, 122 insertions(+), 91 deletions(-) diff --git a/msa/js-executor/api/httpServer.ts b/msa/js-executor/api/httpServer.ts index f3671a1b89..e1c294fdff 100644 --- a/msa/js-executor/api/httpServer.ts +++ b/msa/js-executor/api/httpServer.ts @@ -16,12 +16,15 @@ import express from 'express'; import { _logger} from '../config/logger'; +import http from 'http'; +import { Socket } from 'net'; export class HttpServer { private logger = _logger('httpServer'); private app = express(); - private server; + private server: http.Server | null; + private connections: Socket[] = []; constructor(httpPort: number) { this.app.get('/livenessProbe', async (req, res) => { @@ -32,15 +35,31 @@ export class HttpServer { }) this.server = this.app.listen(httpPort, () => { - this.logger.info('Started http endpoint on port %s. Please, use /livenessProbe !', httpPort); + this.logger.info('Started HTTP endpoint on port %s. Please, use /livenessProbe !', httpPort); }).on('error', (error) => { this.logger.error(error); }); - } - stop() { - this.server.close(() => { - this.logger.info('Http server stop'); + this.server.on('connection', connection => { + this.connections.push(connection); + connection.on('close', () => this.connections = this.connections.filter(curr => curr !== connection)); }); } + + async stop() { + if (this.server) { + this.logger.info('Stopping HTTP Server...'); + const _server = this.server; + this.server = null; + this.connections.forEach(curr => curr.end(() => curr.destroy())); + await new Promise( + (resolve, reject) => { + _server.close((err) => { + this.logger.info('HTTP Server stopped.'); + resolve(); + }); + } + ); + } + } } diff --git a/msa/js-executor/queue/awsSqsTemplate.ts b/msa/js-executor/queue/awsSqsTemplate.ts index 7bbf2b28fc..28d421269b 100644 --- a/msa/js-executor/queue/awsSqsTemplate.ts +++ b/msa/js-executor/queue/awsSqsTemplate.ts @@ -59,8 +59,6 @@ export class AwsSqsTemplate implements IQueue { async init() { try { - this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - this.sqsClient = new SQSClient({ apiVersion: '2012-11-05', credentials: { @@ -129,7 +127,7 @@ export class AwsSqsTemplate implements IQueue { } catch (e: any) { this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); this.logger.error(e.stack); - await this.exit(-1); + await this.destroy(-1); } } @@ -193,23 +191,23 @@ export class AwsSqsTemplate implements IQueue { return queue; } - async exit(status: number) { + async destroy(status: number): Promise { this.stopped = true; this.logger.info('Exiting with status: %d ...', status); + this.logger.info('Stopping AWS SQS resources...'); if (this.sqsClient) { - this.logger.info('Stopping Aws Sqs client.') + this.logger.info('Stopping AWS SQS client...'); try { - this.sqsClient.destroy(); + const _sqsClient = this.sqsClient; // @ts-ignore delete this.sqsClient; - this.logger.info('Aws Sqs client stopped.') - process.exit(status); + _sqsClient.destroy(); + this.logger.info('AWS SQS client stopped.'); } catch (e: any) { - this.logger.info('Aws Sqs client stop error.'); - process.exit(status); + this.logger.info('AWS SQS client stop error.'); } - } else { - process.exit(status); } + this.logger.info('AWS SQS resources stopped.') + process.exit(status); } } diff --git a/msa/js-executor/queue/kafkaTemplate.ts b/msa/js-executor/queue/kafkaTemplate.ts index 8c7e7736c8..2b3e947b5e 100644 --- a/msa/js-executor/queue/kafkaTemplate.ts +++ b/msa/js-executor/queue/kafkaTemplate.ts @@ -56,8 +56,6 @@ export class KafkaTemplate implements IQueue { async init(): Promise { try { - this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - const kafkaBootstrapServers: string = config.get('kafka.bootstrap.servers'); const requestTopic: string = config.get('request_topic'); const useConfluent = config.get('kafka.use_confluent_cloud'); @@ -119,11 +117,11 @@ export class KafkaTemplate implements IQueue { const {CRASH} = this.consumer.events; - this.consumer.on(CRASH, e => { + this.consumer.on(CRASH, async (e) => { this.logger.error(`Got consumer CRASH event, should restart: ${e.payload.restart}`); if (!e.payload.restart) { this.logger.error('Going to exit due to not retryable error!'); - this.exit(-1); + await this.destroy(-1); } }); @@ -133,7 +131,6 @@ export class KafkaTemplate implements IQueue { this.sendLoopWithLinger(); await this.consumer.subscribe({topic: requestTopic}); - this.logger.info('Started ThingsBoard JavaScript Executor Microservice.'); await this.consumer.run({ partitionsConsumedConcurrently: this.partitionsConsumedConcurrently, eachMessage: async ({topic, partition, message}) => { @@ -153,7 +150,7 @@ export class KafkaTemplate implements IQueue { } catch (e: any) { this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); this.logger.error(e.stack); - await this.exit(-1); + await this.destroy(-1); } } @@ -242,34 +239,35 @@ export class KafkaTemplate implements IQueue { } - async exit(status: number): Promise { + async destroy(status: number): Promise { this.logger.info('Exiting with status: %d ...', status); + this.logger.info('Stopping Kafka resources...'); if (this.kafkaAdmin) { this.logger.info('Stopping Kafka Admin...'); - await this.kafkaAdmin.disconnect(); + const _kafkaAdmin = this.kafkaAdmin; // @ts-ignore delete this.kafkaAdmin; + await _kafkaAdmin.disconnect(); this.logger.info('Kafka Admin stopped.'); } if (this.consumer) { this.logger.info('Stopping Kafka Consumer...'); try { - await this.consumer.disconnect(); + const _consumer = this.consumer; // @ts-ignore delete this.consumer; + await _consumer.disconnect(); this.logger.info('Kafka Consumer stopped.'); await this.disconnectProducer(); - process.exit(status); } catch (e: any) { this.logger.info('Kafka Consumer stop error.'); await this.disconnectProducer(); - process.exit(status); } - } else { - process.exit(status); } + this.logger.info('Kafka resources stopped.'); + process.exit(status); } private async disconnectProducer(): Promise { @@ -279,9 +277,10 @@ export class KafkaTemplate implements IQueue { this.logger.info('Stopping loop...'); clearTimeout(this.sendLoopInstance); await this.sendMessagesAsBatch(); - await this.producer.disconnect(); + const _producer = this.producer; // @ts-ignore delete this.producer; + await _producer.disconnect(); this.logger.info('Kafka Producer stopped.'); } catch (e) { this.logger.info('Kafka Producer stop error.'); diff --git a/msa/js-executor/queue/pubSubTemplate.ts b/msa/js-executor/queue/pubSubTemplate.ts index f14aaf5771..4e8990a105 100644 --- a/msa/js-executor/queue/pubSubTemplate.ts +++ b/msa/js-executor/queue/pubSubTemplate.ts @@ -39,7 +39,6 @@ export class PubSubTemplate implements IQueue { async init() { try { - this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); this.pubSubClient = new PubSub({ projectId: this.projectId, credentials: this.credentials @@ -82,7 +81,7 @@ export class PubSubTemplate implements IQueue { } catch (e: any) { this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); this.logger.error(e.stack); - await this.exit(-1); + await this.destroy(-1); } } @@ -153,23 +152,23 @@ export class PubSubTemplate implements IQueue { return queue; } - async exit(status: number): Promise { + async destroy(status: number): Promise { this.logger.info('Exiting with status: %d ...', status); + this.logger.info('Stopping Pub/Sub resources...'); if (this.pubSubClient) { - this.logger.info('Stopping Pub/Sub client.') + this.logger.info('Stopping Pub/Sub client...'); try { - await this.pubSubClient.close(); + const _pubSubClient = this.pubSubClient; // @ts-ignore delete this.pubSubClient; - this.logger.info('Pub/Sub client stopped.') - process.exit(status); + await _pubSubClient.close(); + this.logger.info('Pub/Sub client stopped.'); } catch (e) { this.logger.info('Pub/Sub client stop error.'); - process.exit(status); } - } else { - process.exit(status); } + this.logger.info('Pub/Sub resources stopped.'); + process.exit(status); } } diff --git a/msa/js-executor/queue/queue.models.ts b/msa/js-executor/queue/queue.models.ts index 18ce1f06b6..59ec68896d 100644 --- a/msa/js-executor/queue/queue.models.ts +++ b/msa/js-executor/queue/queue.models.ts @@ -17,5 +17,5 @@ export interface IQueue { init(): Promise; send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise; - exit(status: number): Promise; + destroy(status: number): Promise; } diff --git a/msa/js-executor/queue/rabbitmqTemplate.ts b/msa/js-executor/queue/rabbitmqTemplate.ts index d18ba80be7..372024a4f3 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.ts +++ b/msa/js-executor/queue/rabbitmqTemplate.ts @@ -49,8 +49,6 @@ export class RabbitMqTemplate implements IQueue { async init(): Promise { try { - this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - const url = `amqp://${this.username}:${this.password}@${this.host}:${this.port}${this.vhost}`; this.connection = await amqp.connect(url); this.channel = await this.connection.createConfirmChannel(); @@ -78,7 +76,7 @@ export class RabbitMqTemplate implements IQueue { } catch (e: any) { this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); this.logger.error(e.stack); - await this.exit(-1); + await this.destroy(-1); } } @@ -120,32 +118,33 @@ export class RabbitMqTemplate implements IQueue { return queue; } - async exit(status: number) { + async destroy(status: number) { this.logger.info('Exiting with status: %d ...', status); + this.logger.info('Stopping RabbitMQ resources...'); if (this.channel) { - this.logger.info('Stopping RabbitMq chanel.') - await this.channel.close(); + this.logger.info('Stopping RabbitMQ chanel...'); + const _channel = this.channel; // @ts-ignore delete this.channel; - this.logger.info('RabbitMq chanel stopped'); + await _channel.close(); + this.logger.info('RabbitMQ chanel stopped'); } if (this.connection) { - this.logger.info('Stopping RabbitMq connection.') + this.logger.info('Stopping RabbitMQ connection...') try { - await this.connection.close(); + const _connection = this.connection; // @ts-ignore delete this.connection; - this.logger.info('RabbitMq client connection.') - process.exit(status); + await _connection.close(); + this.logger.info('RabbitMQ client connection.'); } catch (e) { - this.logger.info('RabbitMq connection stop error.'); - process.exit(status); + this.logger.info('RabbitMQ connection stop error.'); } - } else { - process.exit(status); } + this.logger.info('RabbitMQ resources stopped.') + process.exit(status); } } diff --git a/msa/js-executor/queue/serviceBusTemplate.ts b/msa/js-executor/queue/serviceBusTemplate.ts index 3cfaf3e10c..b2750672e5 100644 --- a/msa/js-executor/queue/serviceBusTemplate.ts +++ b/msa/js-executor/queue/serviceBusTemplate.ts @@ -49,8 +49,6 @@ export class ServiceBusTemplate implements IQueue { async init() { try { - this.logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); - const connectionString = `Endpoint=sb://${this.namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${this.sasKeyName};SharedAccessKey=${this.sasKey}`; this.sbClient = new ServiceBusClient(connectionString) this.serviceBusService = new ServiceBusAdministrationClient(connectionString); @@ -84,7 +82,7 @@ export class ServiceBusTemplate implements IQueue { } catch (e: any) { this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); this.logger.error(e.stack); - await this.exit(-1); + await this.destroy(-1); } } @@ -141,32 +139,45 @@ export class ServiceBusTemplate implements IQueue { return queue; } - async exit(status: number) { + async destroy(status: number) { this.logger.info('Exiting with status: %d ...', status); this.logger.info('Stopping Azure Service Bus resources...') if (this.receiver) { + this.logger.info('Stopping Service Bus Receiver...'); try { - await this.receiver.close(); + const _receiver = this.receiver; // @ts-ignore delete this.receiver; + await _receiver.close(); + this.logger.info('Service Bus Receiver stopped.'); } catch (e) { + this.logger.info('Service Bus Receiver stop error.'); } } - this.senderMap.forEach(k => { - try { - k.close(); - } catch (e) { - } + this.logger.info('Stopping Service Bus Senders...'); + const senders: Promise[] = []; + this.senderMap.forEach((sender) => { + senders.push(sender.close()); }); this.senderMap.clear(); + try { + await Promise.all(senders); + this.logger.info('Service Bus Senders stopped.'); + } catch (e) { + this.logger.info('Service Bus Senders stop error.'); + } if (this.sbClient) { + this.logger.info('Stopping Service Bus Client...'); try { - await this.sbClient.close(); + const _sbClient = this.sbClient; // @ts-ignore delete this.sbClient; + await _sbClient.close(); + this.logger.info('Service Bus Client stopped.'); } catch (e) { + this.logger.info('Service Bus Client stop error.'); } } this.logger.info('Azure Service Bus resources stopped.') diff --git a/msa/js-executor/server.ts b/msa/js-executor/server.ts index 3d59d7fe00..708a87fec9 100644 --- a/msa/js-executor/server.ts +++ b/msa/js-executor/server.ts @@ -32,33 +32,34 @@ logger.info('===CONFIG END==='); const serviceType = config.get('queue_type'); const httpPort = Number(config.get('http_port')); -let queues: IQueue; -let httpServer: HttpServer; +let queues: IQueue | null; +let httpServer: HttpServer | null; (async () => { + logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); switch (serviceType) { case 'kafka': - logger.info('Starting kafka template.'); + logger.info('Starting Kafka template...'); queues = await KafkaTemplate.build(); - logger.info('kafka template started.'); + logger.info('Kafka template started.'); break; case 'pubsub': - logger.info('Starting Pub/Sub template.') + logger.info('Starting Pub/Sub template...') queues = await PubSubTemplate.build(); logger.info('Pub/Sub template started.') break; case 'aws-sqs': - logger.info('Starting Aws Sqs template.') + logger.info('Starting AWS SQS template...') queues = await AwsSqsTemplate.build(); - logger.info('Aws Sqs template started.') + logger.info('AWS SQS template started.') break; case 'rabbitmq': - logger.info('Starting RabbitMq template.') + logger.info('Starting RabbitMQ template...') queues = await RabbitMqTemplate.build(); - logger.info('RabbitMq template started.') + logger.info('RabbitMQ template started.') break; case 'service-bus': - logger.info('Starting Azure Service Bus template.') + logger.info('Starting Azure Service Bus template...') queues = await ServiceBusTemplate.build(); logger.info('Azure Service Bus template started.') break; @@ -70,17 +71,22 @@ let httpServer: HttpServer; httpServer = new HttpServer(httpPort); })(); -process.on('SIGTERM', () => { - logger.info('SIGTERM signal received'); - process.exit(0); -}); +[`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach((eventType) => { + process.on(eventType, async () => { + logger.info(`${eventType} signal received`); + if (httpServer) { + const _httpServer = httpServer; + httpServer = null; + await _httpServer.stop(); + } + if (queues) { + const _queues = queues; + queues = null; + await _queues.destroy(0); + } + }) +}) -process.on('exit', async () => { - if (httpServer) { - httpServer.stop(); - } - if (queues) { - queues.exit(0); - } - logger.info('JavaScript Executor Microservice has been stopped.'); +process.on('exit', (code: number) => { + logger.info(`JavaScript Executor Microservice has been stopped. Exit code: ${code}.`); }); From d3541573b2df8d3eb8237161cfedaa3d450dac36 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Thu, 7 Jul 2022 17:44:23 +0300 Subject: [PATCH 14/28] Fixed Docker file for JavaScript Executor Microservice --- msa/js-executor/docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/msa/js-executor/docker/Dockerfile b/msa/js-executor/docker/Dockerfile index 620d712ad1..d17138d923 100644 --- a/msa/js-executor/docker/Dockerfile +++ b/msa/js-executor/docker/Dockerfile @@ -29,6 +29,7 @@ COPY package/linux/conf ./conf COPY package/linux/conf ./config COPY src/api ./api COPY src/queue ./queue +COPY src/config ./config COPY src/server.js ./ RUN chmod a+x /tmp/*.sh \ From 5b10aafcd57492ef8abafbbd8cd00cc3b1062074 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Thu, 7 Jul 2022 18:37:08 +0300 Subject: [PATCH 15/28] refactoring: tests Tenant --- .../controller/AbstractNotifyEntityTest.java | 6 +- .../controller/BaseTenantControllerTest.java | 82 +++++++++++++++++-- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index a32ec3c427..87d41b8e2f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -381,7 +381,9 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } protected void testBroadcastEntityStateChangeEventTime(EntityId entityId, TenantId tenantId, int cntTime) { - Mockito.verify(tbClusterService, times(cntTime)).broadcastEntityStateChangeEvent(Mockito.eq(tenantId), + ArgumentMatcher matcherTenantIdId = cntTime == 1 ? argument -> argument.equals(tenantId) : + argument -> argument.getClass().equals(TenantId.class); + Mockito.verify(tbClusterService, times(cntTime)).broadcastEntityStateChangeEvent(Mockito.argThat(matcherTenantIdId), Mockito.any(entityId.getClass()), Mockito.any(ComponentLifecycleEvent.class)); } @@ -612,7 +614,7 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { return result; } - private EntityId createEntityId_NULL_UUID(HasName entity) { + protected EntityId createEntityId_NULL_UUID(HasName entity) { return EntityIdFactory.getByTypeAndUuid(entityClassToEntityTypeName(entity), ModelConstants.NULL_UUID); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java index 9a12653d16..a59d8612d1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java @@ -26,6 +26,8 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.ArgumentMatcher; +import org.mockito.Mockito; import org.springframework.test.context.TestPropertySource; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; @@ -33,8 +35,10 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantInfo; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.ProcessingStrategy; import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.Queue; @@ -56,6 +60,8 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @TestPropertySource(properties = { @@ -86,17 +92,28 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); + + Mockito.reset(tbClusterService, auditLogService); + Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); Assert.assertNotNull(savedTenant); Assert.assertNotNull(savedTenant.getId()); Assert.assertTrue(savedTenant.getCreatedTime() > 0); Assert.assertEquals(tenant.getTitle(), savedTenant.getTitle()); + + testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.CREATED, 1); + savedTenant.setTitle("My new tenant"); doPost("/api/tenant", savedTenant, Tenant.class); Tenant foundTenant = doGet("/api/tenant/" + savedTenant.getId().getId().toString(), Tenant.class); Assert.assertEquals(foundTenant.getTitle(), savedTenant.getTitle()); + + testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.UPDATED, 1); + doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) .andExpect(status().isOk()); + + testBroadcastEntityStateChangeEventTimeManyTimeTenant(savedTenant, ComponentLifecycleEvent.DELETED, 1); } @Test @@ -104,7 +121,14 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { loginSysAdmin(); Tenant tenant = new Tenant(); tenant.setTitle(RandomStringUtils.randomAlphanumeric(300)); - doPost("/api/tenant", tenant).andExpect(statusReason(containsString("length of title must be equal or less than 255"))); + + Mockito.reset(tbClusterService, auditLogService); + + doPost("/api/tenant", tenant) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgErrorFieldLength("title")))); + + testBroadcastEntityStateChangeEventNeverTenant(); } @Test @@ -136,21 +160,31 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { @Test public void testSaveTenantWithEmptyTitle() throws Exception { loginSysAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Tenant tenant = new Tenant(); doPost("/api/tenant", tenant) .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Tenant title should be specified"))); + .andExpect(statusReason(containsString("Tenant title " + msgErrorShouldBeSpecified))); + + testBroadcastEntityStateChangeEventNeverTenant(); } @Test public void testSaveTenantWithInvalidEmail() throws Exception { loginSysAdmin(); + + Mockito.reset(tbClusterService, auditLogService); + Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); tenant.setEmail("invalid@mail"); doPost("/api/tenant", tenant) .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString("Invalid email address format"))); + + testBroadcastEntityStateChangeEventNeverTenant(); } @Test @@ -159,10 +193,13 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); - doDelete("/api/tenant/" + savedTenant.getId().getId().toString()) + + String tenantIdStr = savedTenant.getId().getId().toString(); + doDelete("/api/tenant/" + tenantIdStr) .andExpect(status().isOk()); - doGet("/api/tenant/" + savedTenant.getId().getId().toString()) - .andExpect(status().isNotFound()); + doGet("/api/tenant/" + tenantIdStr) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Tenant", tenantIdStr)))); } @Test @@ -175,8 +212,11 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { Assert.assertEquals(1, pageData.getData().size()); tenants.addAll(pageData.getData()); + Mockito.reset(tbClusterService, auditLogService); + + int cntEntity = 56; List> createFutures = new ArrayList<>(56); - for (int i = 0; i < 56; i++) { + for (int i = 0; i < cntEntity; i++) { Tenant tenant = new Tenant(); tenant.setTitle("Tenant" + i); createFutures.add(executor.submit(() -> @@ -184,6 +224,8 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { } tenants.addAll(Futures.allAsList(createFutures).get(TIMEOUT, TimeUnit.SECONDS)); + testBroadcastEntityStateChangeEventTimeManyTimeTenant(new Tenant(), ComponentLifecycleEvent.CREATED, cntEntity); + List loadedTenants = new ArrayList<>(); pageLink = new PageLink(17); do { @@ -200,6 +242,8 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { .filter((t) -> !TEST_TENANT_NAME.equals(t.getTitle())) .collect(Collectors.toList()), executor).get(TIMEOUT, TimeUnit.SECONDS); + testBroadcastEntityStateChangeEventTimeManyTimeTenant(new Tenant(), ComponentLifecycleEvent.DELETED, cntEntity); + pageLink = new PageLink(17); pageData = doGetTypedWithPageLink("/api/tenants?", PAGE_DATA_TENANT_TYPE_REF, pageLink); Assert.assertFalse(pageData.hasNext()); @@ -464,7 +508,9 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { login(username, password); for (Queue queue : foundTenantQueues) { - doGet("/api/queues/" + queue.getId()).andExpect(status().isNotFound()); + doGet("/api/queues/" + queue.getId()) + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNotFound))); } loginSysAdmin(); @@ -519,4 +565,26 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { } return result; } + + private void testBroadcastEntityStateChangeEventTimeManyTimeTenant(Tenant tenant, ComponentLifecycleEvent event, int cntTime) { + ArgumentMatcher matcherTenant = cntTime == 1 ? argument -> argument.equals(tenant) : + argument -> argument.getClass().equals(Tenant.class); + if (ComponentLifecycleEvent.DELETED.equals(event)) { + Mockito.verify(tbClusterService, times( cntTime)).onTenantDelete(Mockito.argThat(matcherTenant), + Mockito.isNull()); + } else { + Mockito.verify(tbClusterService, times( cntTime)).onTenantChange(Mockito.argThat(matcherTenant), + Mockito.isNull()); + } + TenantId tenantId = cntTime == 1 ? tenant.getId() : (TenantId) createEntityId_NULL_UUID(tenant); + testBroadcastEntityStateChangeEventTime(tenantId, tenantId, cntTime); + Mockito.reset(tbClusterService, auditLogService); + } + + private void testBroadcastEntityStateChangeEventNeverTenant() { + Mockito.verify(tbClusterService, never()).onTenantChange(Mockito.any(Tenant.class), + Mockito.isNull()); + testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); + Mockito.reset(tbClusterService, auditLogService); + } } From f7225f643c60ab5be719ec6ecc078d337dd772a8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 7 Jul 2022 18:41:36 +0300 Subject: [PATCH 16/28] Fix Queue settings for AWS --- .../src/main/resources/thingsboard.yml | 6 ++++ .../servicebus/TbServiceBusQueueConfigs.java | 23 +++++++------ .../queue/kafka/TbKafkaTopicConfigs.java | 10 +++--- .../provider/AwsSqsMonolithQueueFactory.java | 9 +++-- .../provider/AwsSqsTbCoreQueueFactory.java | 21 +++++++++--- .../AwsSqsTbRuleEngineQueueFactory.java | 8 ++++- .../pubsub/TbPubSubSubscriptionSettings.java | 15 +++++---- .../rabbitmq/TbRabbitMqQueueArguments.java | 23 +++++++------ .../queue/sqs/TbAwsSqsQueueAttributes.java | 33 +++++++++++-------- .../src/main/resources/tb-vc-executor.yml | 21 +++--------- .../src/main/resources/tb-coap-transport.yml | 5 --- .../src/main/resources/tb-http-transport.yml | 5 --- .../src/main/resources/tb-lwm2m-transport.yml | 5 --- .../src/main/resources/tb-mqtt-transport.yml | 5 --- .../src/main/resources/tb-snmp-transport.yml | 5 --- 15 files changed, 102 insertions(+), 92 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9b458734f7..a6fe1b97f0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -949,6 +949,7 @@ queue: notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" consumer-stats: enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" @@ -965,6 +966,8 @@ queue: transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + ota-updates: "${TB_QUEUE_AWS_SQS_OTA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + version-control: "${TB_QUEUE_AWS_SQS_VC_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" @@ -977,6 +980,7 @@ queue: transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + version-control: "${TB_QUEUE_PUBSUB_VC_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" @@ -988,6 +992,7 @@ queue: transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + version-control: "${TB_QUEUE_SERVICE_BUS_VC_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" @@ -1004,6 +1009,7 @@ queue: transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + version-control: "${TB_QUEUE_RABBIT_MQ_VC_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 transport_api: diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java index b87049b634..c2974c3e3a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/azure/servicebus/TbServiceBusQueueConfigs.java @@ -19,6 +19,7 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.StringUtils; import javax.annotation.PostConstruct; import java.util.HashMap; @@ -27,15 +28,15 @@ import java.util.Map; @Component @ConditionalOnExpression("'${queue.type:null}'=='service-bus'") public class TbServiceBusQueueConfigs { - @Value("${queue.service-bus.queue-properties.core}") + @Value("${queue.service-bus.queue-properties.core:}") private String coreProperties; - @Value("${queue.service-bus.queue-properties.rule-engine}") + @Value("${queue.service-bus.queue-properties.rule-engine:}") private String ruleEngineProperties; - @Value("${queue.service-bus.queue-properties.transport-api}") + @Value("${queue.service-bus.queue-properties.transport-api:}") private String transportApiProperties; - @Value("${queue.service-bus.queue-properties.notifications}") + @Value("${queue.service-bus.queue-properties.notifications:}") private String notificationsProperties; - @Value("${queue.service-bus.queue-properties.js-executor}") + @Value("${queue.service-bus.queue-properties.js-executor:}") private String jsExecutorProperties; @Value("${queue.service-bus.queue-properties.version-control:}") private String vcProperties; @@ -64,11 +65,13 @@ public class TbServiceBusQueueConfigs { private Map getConfigs(String properties) { Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); + if (StringUtils.isNotEmpty(properties)) { + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } } return configs; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java index 75d0bd9b2d..9e768b2b14 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaTopicConfigs.java @@ -28,15 +28,15 @@ import java.util.Map; @Component @ConditionalOnProperty(prefix = "queue", value = "type", havingValue = "kafka") public class TbKafkaTopicConfigs { - @Value("${queue.kafka.topic-properties.core}") + @Value("${queue.kafka.topic-properties.core:}") private String coreProperties; - @Value("${queue.kafka.topic-properties.rule-engine}") + @Value("${queue.kafka.topic-properties.rule-engine:}") private String ruleEngineProperties; - @Value("${queue.kafka.topic-properties.transport-api}") + @Value("${queue.kafka.topic-properties.transport-api:}") private String transportApiProperties; - @Value("${queue.kafka.topic-properties.notifications}") + @Value("${queue.kafka.topic-properties.notifications:}") private String notificationsProperties; - @Value("${queue.kafka.topic-properties.js-executor}") + @Value("${queue.kafka.topic-properties.js-executor:}") private String jsExecutorProperties; @Value("${queue.kafka.topic-properties.ota-updates:}") private String fwUpdatesProperties; diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java index 2ca5090aed..41efe08c1c 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsMonolithQueueFactory.java @@ -76,6 +76,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin otaAdmin; private final TbQueueAdmin vcAdmin; public AwsSqsMonolithQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, @@ -102,6 +103,7 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + this.otaAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getOtaAttributes()); this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); } @@ -210,13 +212,13 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng @Override public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic(), + return new TbAwsSqsConsumerTemplate<>(otaAdmin, sqsSettings, coreSettings.getOtaPackageTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); + return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } @Override @@ -241,6 +243,9 @@ public class AwsSqsMonolithQueueFactory implements TbCoreQueueFactory, TbRuleEng if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (otaAdmin != null) { + otaAdmin.destroy(); + } if (vcAdmin != null) { vcAdmin.destroy(); } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java index 89b45e2822..17dcd07919 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbCoreQueueFactory.java @@ -45,6 +45,7 @@ import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportApiSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -66,18 +67,22 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { private final TbServiceInfoProvider serviceInfoProvider; private final TbQueueRemoteJsInvokeSettings jsInvokeSettings; private final TbQueueTransportNotificationSettings transportNotificationSettings; + private final TbQueueVersionControlSettings vcSettings; private final TbQueueAdmin coreAdmin; private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin transportApiAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin otaAdmin; + private final TbQueueAdmin vcAdmin; public AwsSqsTbCoreQueueFactory(TbAwsSqsSettings sqsSettings, TbQueueCoreSettings coreSettings, TbQueueTransportApiSettings transportApiSettings, TbQueueRuleEngineSettings ruleEngineSettings, NotificationsTopicService notificationsTopicService, + TbQueueVersionControlSettings vcSettings, TbServiceInfoProvider serviceInfoProvider, TbQueueRemoteJsInvokeSettings jsInvokeSettings, TbAwsSqsQueueAttributes sqsQueueAttributes, @@ -90,12 +95,15 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { this.serviceInfoProvider = serviceInfoProvider; this.jsInvokeSettings = jsInvokeSettings; this.transportNotificationSettings = transportNotificationSettings; + this.vcSettings = vcSettings; this.coreAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getCoreAttributes()); this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); this.transportApiAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getTransportApiAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + this.otaAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getOtaAttributes()); + this.vcAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getVcAttributes()); } @Override @@ -183,19 +191,18 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { @Override public TbQueueConsumer> createToOtaPackageStateServiceMsgConsumer() { - return new TbAwsSqsConsumerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic(), + return new TbAwsSqsConsumerTemplate<>(otaAdmin, sqsSettings, coreSettings.getOtaPackageTopic(), msg -> new TbProtoQueueMsg<>(msg.getKey(), ToOtaPackageStateServiceMsg.parseFrom(msg.getData()), msg.getHeaders())); } @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); + return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } @Override public TbQueueProducer> createVersionControlMsgProducer() { - //TODO: version-control - return null; + return new TbAwsSqsProducerTemplate<>(vcAdmin, sqsSettings, vcSettings.getTopic()); } @PreDestroy @@ -215,5 +222,11 @@ public class AwsSqsTbCoreQueueFactory implements TbCoreQueueFactory { if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (otaAdmin != null) { + otaAdmin.destroy(); + } + if (vcAdmin != null) { + vcAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java index 9d9fd29078..94df62eb3a 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/AwsSqsTbRuleEngineQueueFactory.java @@ -39,6 +39,7 @@ import org.thingsboard.server.queue.settings.TbQueueCoreSettings; import org.thingsboard.server.queue.settings.TbQueueRemoteJsInvokeSettings; import org.thingsboard.server.queue.settings.TbQueueRuleEngineSettings; import org.thingsboard.server.queue.settings.TbQueueTransportNotificationSettings; +import org.thingsboard.server.queue.settings.TbQueueVersionControlSettings; import org.thingsboard.server.queue.sqs.TbAwsSqsAdmin; import org.thingsboard.server.queue.sqs.TbAwsSqsConsumerTemplate; import org.thingsboard.server.queue.sqs.TbAwsSqsProducerTemplate; @@ -64,6 +65,7 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory private final TbQueueAdmin ruleEngineAdmin; private final TbQueueAdmin jsExecutorAdmin; private final TbQueueAdmin notificationAdmin; + private final TbQueueAdmin otaAdmin; public AwsSqsTbRuleEngineQueueFactory(NotificationsTopicService notificationsTopicService, TbQueueCoreSettings coreSettings, TbQueueRuleEngineSettings ruleEngineSettings, @@ -84,6 +86,7 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory this.ruleEngineAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getRuleEngineAttributes()); this.jsExecutorAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getJsExecutorAttributes()); this.notificationAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getNotificationsAttributes()); + this.otaAdmin = new TbAwsSqsAdmin(sqsSettings, sqsQueueAttributes.getOtaAttributes()); } @Override @@ -154,7 +157,7 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory @Override public TbQueueProducer> createToOtaPackageStateServiceMsgProducer() { - return new TbAwsSqsProducerTemplate<>(coreAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); + return new TbAwsSqsProducerTemplate<>(otaAdmin, sqsSettings, coreSettings.getOtaPackageTopic()); } @@ -172,6 +175,9 @@ public class AwsSqsTbRuleEngineQueueFactory implements TbRuleEngineQueueFactory if (notificationAdmin != null) { notificationAdmin.destroy(); } + if (otaAdmin != null) { + otaAdmin.destroy(); + } } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java index a4819fdad4..a586e4f6fa 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/pubsub/TbPubSubSubscriptionSettings.java @@ -19,6 +19,7 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.StringUtils; import javax.annotation.PostConstruct; import java.util.HashMap; @@ -27,7 +28,7 @@ import java.util.Map; @Component @ConditionalOnExpression("'${queue.type:null}'=='pubsub'") public class TbPubSubSubscriptionSettings { - @Value("${queue.pubsub.queue-properties.core}") + @Value("${queue.pubsub.queue-properties.core:}") private String coreProperties; @Value("${queue.pubsub.queue-properties.rule-engine}") private String ruleEngineProperties; @@ -65,11 +66,13 @@ public class TbPubSubSubscriptionSettings { private Map getSettings(String properties) { Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - configs.put(key, value); + if (StringUtils.isNotEmpty(properties)) { + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + configs.put(key, value); + } } return configs; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java index 25c78a96c5..af3602fd73 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/rabbitmq/TbRabbitMqQueueArguments.java @@ -19,6 +19,7 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.StringUtils; import javax.annotation.PostConstruct; import java.util.HashMap; @@ -28,15 +29,15 @@ import java.util.regex.Pattern; @Component @ConditionalOnExpression("'${queue.type:null}'=='rabbitmq'") public class TbRabbitMqQueueArguments { - @Value("${queue.rabbitmq.queue-properties.core}") + @Value("${queue.rabbitmq.queue-properties.core:}") private String coreProperties; - @Value("${queue.rabbitmq.queue-properties.rule-engine}") + @Value("${queue.rabbitmq.queue-properties.rule-engine:}") private String ruleEngineProperties; - @Value("${queue.rabbitmq.queue-properties.transport-api}") + @Value("${queue.rabbitmq.queue-properties.transport-api:}") private String transportApiProperties; - @Value("${queue.rabbitmq.queue-properties.notifications}") + @Value("${queue.rabbitmq.queue-properties.notifications:}") private String notificationsProperties; - @Value("${queue.rabbitmq.queue-properties.js-executor}") + @Value("${queue.rabbitmq.queue-properties.js-executor:}") private String jsExecutorProperties; @Value("${queue.rabbitmq.queue-properties.version-control:}") private String vcProperties; @@ -66,11 +67,13 @@ public class TbRabbitMqQueueArguments { private Map getArgs(String properties) { Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String strValue = property.substring(delimiterPosition + 1); - configs.put(key, getObjectValue(strValue)); + if (StringUtils.isNotEmpty(properties)) { + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String strValue = property.substring(delimiterPosition + 1); + configs.put(key, getObjectValue(strValue)); + } } return configs; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java index 58704f62a7..2e28b56076 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsQueueAttributes.java @@ -20,6 +20,7 @@ import lombok.Getter; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.StringUtils; import javax.annotation.PostConstruct; import java.util.HashMap; @@ -28,16 +29,18 @@ import java.util.Map; @Component @ConditionalOnExpression("'${queue.type:null}'=='aws-sqs'") public class TbAwsSqsQueueAttributes { - @Value("${queue.aws-sqs.queue-properties.core}") + @Value("${queue.aws-sqs.queue-properties.core:}") private String coreProperties; - @Value("${queue.aws-sqs.queue-properties.rule-engine}") + @Value("${queue.aws-sqs.queue-properties.rule-engine:}") private String ruleEngineProperties; - @Value("${queue.aws-sqs.queue-properties.transport-api}") + @Value("${queue.aws-sqs.queue-properties.transport-api:}") private String transportApiProperties; - @Value("${queue.aws-sqs.queue-properties.notifications}") + @Value("${queue.aws-sqs.queue-properties.notifications:}") private String notificationsProperties; - @Value("${queue.aws-sqs.queue-properties.js-executor}") + @Value("${queue.aws-sqs.queue-properties.js-executor:}") private String jsExecutorProperties; + @Value("${queue.aws-sqs.queue-properties.ota-updates:}") + private String otaProperties; @Value("${queue.aws-sqs.queue-properties.version-control:}") private String vcProperties; @@ -52,6 +55,8 @@ public class TbAwsSqsQueueAttributes { @Getter private Map jsExecutorAttributes; @Getter + private Map otaAttributes; + @Getter private Map vcAttributes; private final Map defaultAttributes = new HashMap<>(); @@ -65,19 +70,21 @@ public class TbAwsSqsQueueAttributes { transportApiAttributes = getConfigs(transportApiProperties); notificationsAttributes = getConfigs(notificationsProperties); jsExecutorAttributes = getConfigs(jsExecutorProperties); + otaAttributes = getConfigs(otaProperties); vcAttributes = getConfigs(vcProperties); } private Map getConfigs(String properties) { - Map configs = new HashMap<>(); - for (String property : properties.split(";")) { - int delimiterPosition = property.indexOf(":"); - String key = property.substring(0, delimiterPosition); - String value = property.substring(delimiterPosition + 1); - validateAttributeName(key); - configs.put(key, value); + Map configs = new HashMap<>(defaultAttributes); + if (StringUtils.isNotEmpty(properties)) { + for (String property : properties.split(";")) { + int delimiterPosition = property.indexOf(":"); + String key = property.substring(0, delimiterPosition); + String value = property.substring(delimiterPosition + 1); + validateAttributeName(key); + configs.put(key, value); + } } - configs.putAll(defaultAttributes); return configs; } diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 1344ea194a..916eb1695f 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -87,12 +87,9 @@ queue: - key: "session.timeout.ms" # refer to https://docs.confluent.io/platform/current/installation/configuration/consumer-configs.html#consumerconfigs_session.timeout.ms value: "${TB_QUEUE_KAFKA_SESSION_TIMEOUT_MS:10000}" # (10 seconds) topic-properties: - rule-engine: "${TB_QUEUE_KAFKA_RE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" - ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}" + version-control: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" consumer-stats: enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" @@ -104,11 +101,9 @@ queue: region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}" threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}" queue-properties: - rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" + version-control: "${TB_QUEUE_AWS_SQS_VC_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" @@ -116,22 +111,18 @@ queue: max_msg_size: "${TB_QUEUE_PUBSUB_MAX_MSG_SIZE:1048576}" #in bytes max_messages: "${TB_QUEUE_PUBSUB_MAX_MESSAGES:1000}" queue-properties: - rule-engine: "${TB_QUEUE_PUBSUB_RE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" + version-control: "${TB_QUEUE_PUBSUB_VC_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" sas_key: "${TB_QUEUE_SERVICE_BUS_SAS_KEY:YOUR_SAS_KEY}" max_messages: "${TB_QUEUE_SERVICE_BUS_MAX_MESSAGES:1000}" queue-properties: - rule-engine: "${TB_QUEUE_SERVICE_BUS_RE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" + version-control: "${TB_QUEUE_SERVICE_BUS_VC_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" @@ -143,11 +134,9 @@ queue: connection_timeout: "${TB_QUEUE_RABBIT_MQ_CONNECTION_TIMEOUT:60000}" handshake_timeout: "${TB_QUEUE_RABBIT_MQ_HANDSHAKE_TIMEOUT:10000}" queue-properties: - rule-engine: "${TB_QUEUE_RABBIT_MQ_RE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" + version-control: "${TB_QUEUE_RABBIT_MQ_VC_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" # murmur3_32, murmur3_128 or sha256 core: diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 2d408e0008..5ef164bd8b 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -174,7 +174,6 @@ queue: core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" @@ -186,7 +185,6 @@ queue: core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" @@ -198,7 +196,6 @@ queue: core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" @@ -209,7 +206,6 @@ queue: core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" @@ -225,7 +221,6 @@ queue: core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 8e8cbc0641..0446a6e27b 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -161,7 +161,6 @@ queue: core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" @@ -173,7 +172,6 @@ queue: core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" @@ -185,7 +183,6 @@ queue: core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" @@ -196,7 +193,6 @@ queue: core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" @@ -212,7 +208,6 @@ queue: core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index fb273a6118..df135372ba 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -239,7 +239,6 @@ queue: core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" @@ -251,7 +250,6 @@ queue: core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" @@ -263,7 +261,6 @@ queue: core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" @@ -274,7 +271,6 @@ queue: core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" @@ -290,7 +286,6 @@ queue: core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 3514449861..e889752ab3 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -191,7 +191,6 @@ queue: core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" aws_sqs: use_default_credential_provider_chain: "${TB_QUEUE_AWS_SQS_USE_DEFAULT_CREDENTIAL_PROVIDER_CHAIN:false}" access_key_id: "${TB_QUEUE_AWS_SQS_ACCESS_KEY_ID:YOUR_KEY}" @@ -203,7 +202,6 @@ queue: core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" @@ -215,7 +213,6 @@ queue: core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" @@ -226,7 +223,6 @@ queue: core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" @@ -242,7 +238,6 @@ queue: core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 5f95ad382d..c20c8e2391 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -137,7 +137,6 @@ queue: core: "${TB_QUEUE_KAFKA_CORE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" transport-api: "${TB_QUEUE_KAFKA_TA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" notifications: "${TB_QUEUE_KAFKA_NOTIFICATIONS_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}" - js-executor: "${TB_QUEUE_KAFKA_JE_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:104857600;partitions:100;min.insync.replicas:1}" consumer-stats: enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}" print-interval-ms: "${TB_QUEUE_KAFKA_CONSUMER_STATS_MIN_PRINT_INTERVAL_MS:60000}" @@ -153,7 +152,6 @@ queue: core: "${TB_QUEUE_AWS_SQS_CORE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" transport-api: "${TB_QUEUE_AWS_SQS_TA_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" notifications: "${TB_QUEUE_AWS_SQS_NOTIFICATIONS_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" - js-executor: "${TB_QUEUE_AWS_SQS_JE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}" # VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds pubsub: project_id: "${TB_QUEUE_PUBSUB_PROJECT_ID:YOUR_PROJECT_ID}" @@ -165,7 +163,6 @@ queue: core: "${TB_QUEUE_PUBSUB_CORE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" transport-api: "${TB_QUEUE_PUBSUB_TA_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" notifications: "${TB_QUEUE_PUBSUB_NOTIFICATIONS_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" - js-executor: "${TB_QUEUE_PUBSUB_JE_QUEUE_PROPERTIES:ackDeadlineInSec:30;messageRetentionInSec:604800}" service_bus: namespace_name: "${TB_QUEUE_SERVICE_BUS_NAMESPACE_NAME:YOUR_NAMESPACE_NAME}" sas_key_name: "${TB_QUEUE_SERVICE_BUS_SAS_KEY_NAME:YOUR_SAS_KEY_NAME}" @@ -176,7 +173,6 @@ queue: core: "${TB_QUEUE_SERVICE_BUS_CORE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" transport-api: "${TB_QUEUE_SERVICE_BUS_TA_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" notifications: "${TB_QUEUE_SERVICE_BUS_NOTIFICATIONS_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" - js-executor: "${TB_QUEUE_SERVICE_BUS_JE_QUEUE_PROPERTIES:lockDurationInSec:30;maxSizeInMb:1024;messageTimeToLiveInSec:604800}" rabbitmq: exchange_name: "${TB_QUEUE_RABBIT_MQ_EXCHANGE_NAME:}" host: "${TB_QUEUE_RABBIT_MQ_HOST:localhost}" @@ -192,7 +188,6 @@ queue: core: "${TB_QUEUE_RABBIT_MQ_CORE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" transport-api: "${TB_QUEUE_RABBIT_MQ_TA_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" notifications: "${TB_QUEUE_RABBIT_MQ_NOTIFICATIONS_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" - js-executor: "${TB_QUEUE_RABBIT_MQ_JE_QUEUE_PROPERTIES:x-max-length-bytes:1048576000;x-message-ttl:604800000}" partitions: hash_function_name: "${TB_QUEUE_PARTITIONS_HASH_FUNCTION_NAME:murmur3_128}" virtual_nodes_size: "${TB_QUEUE_PARTITIONS_VIRTUAL_NODES_SIZE:16}" From 7019b98c00a9c0a045b446cd571e958762d5bef7 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 7 Jul 2022 19:37:36 +0300 Subject: [PATCH 17/28] JS executor code refactor --- msa/js-executor/package.json | 2 +- msa/js-executor/queue/awsSqsTemplate.ts | 128 +++++++-------- msa/js-executor/queue/kafkaTemplate.ts | 173 +++++++++----------- msa/js-executor/queue/pubSubTemplate.ts | 75 ++++----- msa/js-executor/queue/queue.models.ts | 3 +- msa/js-executor/queue/rabbitmqTemplate.ts | 50 +++--- msa/js-executor/queue/serviceBusTemplate.ts | 70 ++++---- msa/js-executor/server.ts | 77 ++++----- msa/js-executor/yarn.lock | 8 +- 9 files changed, 264 insertions(+), 322 deletions(-) diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 1d1059aa90..6bc7668e15 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -20,7 +20,7 @@ "config": "^3.3.7", "express": "^4.18.1", "js-yaml": "^4.1.0", - "kafkajs": "^2.0.2", + "kafkajs": "^2.1.0", "long": "^5.2.0", "uuid-parse": "^1.1.0", "uuid-random": "^1.3.2", diff --git a/msa/js-executor/queue/awsSqsTemplate.ts b/msa/js-executor/queue/awsSqsTemplate.ts index 28d421269b..36ec9b5270 100644 --- a/msa/js-executor/queue/awsSqsTemplate.ts +++ b/msa/js-executor/queue/awsSqsTemplate.ts @@ -54,80 +54,76 @@ export class AwsSqsTemplate implements IQueue { FifoQueue: 'true' }; + name = 'AWS SQS'; + constructor() { } async init() { - try { - this.sqsClient = new SQSClient({ - apiVersion: '2012-11-05', - credentials: { - accessKeyId: this.accessKeyId, - secretAccessKey: this.secretAccessKey - }, - region: this.region - }); + this.sqsClient = new SQSClient({ + apiVersion: '2012-11-05', + credentials: { + accessKeyId: this.accessKeyId, + secretAccessKey: this.secretAccessKey + }, + region: this.region + }); - const queues = await this.getQueues(); + const queues = await this.getQueues(); - if (queues.QueueUrls) { - queues.QueueUrls.forEach(queueUrl => { - const delimiterPosition = queueUrl.lastIndexOf('/'); - const queueName = queueUrl.substring(delimiterPosition + 1); - this.queueUrls.set(queueName, queueUrl); - }); - } + if (queues.QueueUrls) { + queues.QueueUrls.forEach(queueUrl => { + const delimiterPosition = queueUrl.lastIndexOf('/'); + const queueName = queueUrl.substring(delimiterPosition + 1); + this.queueUrls.set(queueName, queueUrl); + }); + } - this.parseQueueProperties(); + this.parseQueueProperties(); - this.requestQueueURL = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(this.requestTopic)) || ''; - if (!this.requestQueueURL) { - this.requestQueueURL = await this.createQueue(this.requestTopic); - } + this.requestQueueURL = this.queueUrls.get(AwsSqsTemplate.topicToSqsQueueName(this.requestTopic)) || ''; + if (!this.requestQueueURL) { + this.requestQueueURL = await this.createQueue(this.requestTopic); + } + + const messageProcessor = new JsInvokeMessageProcessor(this); - const messageProcessor = new JsInvokeMessageProcessor(this); - - const params: ReceiveMessageRequest = { - MaxNumberOfMessages: 10, - QueueUrl: this.requestQueueURL, - WaitTimeSeconds: this.pollInterval / 1000 - }; - while (!this.stopped) { - let pollStartTs = new Date().getTime(); - const messagesResponse: ReceiveMessageResult = await this.sqsClient.send(new ReceiveMessageCommand(params)); - const messages = messagesResponse.Messages; - - if (messages && messages.length > 0) { - const entries: DeleteMessageBatchRequestEntry[] = []; - - messages.forEach(message => { - entries.push({ - Id: message.MessageId, - ReceiptHandle: message.ReceiptHandle - }); - messageProcessor.onJsInvokeMessage(JSON.parse(message.Body || '')); + const params: ReceiveMessageRequest = { + MaxNumberOfMessages: 10, + QueueUrl: this.requestQueueURL, + WaitTimeSeconds: this.pollInterval / 1000 + }; + while (!this.stopped) { + let pollStartTs = new Date().getTime(); + const messagesResponse: ReceiveMessageResult = await this.sqsClient.send(new ReceiveMessageCommand(params)); + const messages = messagesResponse.Messages; + + if (messages && messages.length > 0) { + const entries: DeleteMessageBatchRequestEntry[] = []; + + messages.forEach(message => { + entries.push({ + Id: message.MessageId, + ReceiptHandle: message.ReceiptHandle }); + messageProcessor.onJsInvokeMessage(JSON.parse(message.Body || '')); + }); - const deleteBatch: DeleteMessageBatchRequest = { - QueueUrl: this.requestQueueURL, - Entries: entries - }; - try { - await this.sqsClient.send(new DeleteMessageBatchCommand(deleteBatch)) - } catch (err: any) { - this.logger.error("Failed to delete messages from queue.", err.message); - } - } else { - let pollDuration = new Date().getTime() - pollStartTs; - if (pollDuration < this.pollInterval) { - await sleep(this.pollInterval - pollDuration); - } + const deleteBatch: DeleteMessageBatchRequest = { + QueueUrl: this.requestQueueURL, + Entries: entries + }; + try { + await this.sqsClient.send(new DeleteMessageBatchCommand(deleteBatch)) + } catch (err: any) { + this.logger.error("Failed to delete messages from queue.", err.message); + } + } else { + let pollDuration = new Date().getTime() - pollStartTs; + if (pollDuration < this.pollInterval) { + await sleep(this.pollInterval - pollDuration); } } - } catch (e: any) { - this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - this.logger.error(e.stack); - await this.destroy(-1); } } @@ -185,15 +181,8 @@ export class AwsSqsTemplate implements IQueue { return result.QueueUrl || ''; } - static async build(): Promise { - const queue = new AwsSqsTemplate(); - await queue.init(); - return queue; - } - - async destroy(status: number): Promise { + async destroy(): Promise { this.stopped = true; - this.logger.info('Exiting with status: %d ...', status); this.logger.info('Stopping AWS SQS resources...'); if (this.sqsClient) { this.logger.info('Stopping AWS SQS client...'); @@ -208,6 +197,5 @@ export class AwsSqsTemplate implements IQueue { } } this.logger.info('AWS SQS resources stopped.') - process.exit(status); } } diff --git a/msa/js-executor/queue/kafkaTemplate.ts b/msa/js-executor/queue/kafkaTemplate.ts index 2b3e947b5e..51fa6e291b 100644 --- a/msa/js-executor/queue/kafkaTemplate.ts +++ b/msa/js-executor/queue/kafkaTemplate.ts @@ -51,108 +51,103 @@ export class KafkaTemplate implements IQueue { private batchMessages: TopicMessages[] = []; private sendLoopInstance: NodeJS.Timeout; + name = 'Kafka'; + constructor() { } async init(): Promise { - try { - const kafkaBootstrapServers: string = config.get('kafka.bootstrap.servers'); - const requestTopic: string = config.get('request_topic'); - const useConfluent = config.get('kafka.use_confluent_cloud'); - - this.logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); - this.logger.info('Kafka Requests Topic: %s', requestTopic); - - let kafkaConfig: KafkaConfig = { - brokers: kafkaBootstrapServers.split(','), - logLevel: logLevel.INFO, - logCreator: KafkaJsWinstonLogCreator - }; + const kafkaBootstrapServers: string = config.get('kafka.bootstrap.servers'); + const requestTopic: string = config.get('request_topic'); + const useConfluent = config.get('kafka.use_confluent_cloud'); - if (this.kafkaClientId) { - kafkaConfig['clientId'] = this.kafkaClientId; - } else { - this.logger.warn('KAFKA_CLIENT_ID is undefined. Consider to define the env variable KAFKA_CLIENT_ID'); - } + this.logger.info('Kafka Bootstrap Servers: %s', kafkaBootstrapServers); + this.logger.info('Kafka Requests Topic: %s', requestTopic); - kafkaConfig['requestTimeout'] = this.requestTimeout; + let kafkaConfig: KafkaConfig = { + brokers: kafkaBootstrapServers.split(','), + logLevel: logLevel.INFO, + logCreator: KafkaJsWinstonLogCreator + }; - if (useConfluent) { - kafkaConfig['sasl'] = { - mechanism: config.get('kafka.confluent.sasl.mechanism') as any, - username: config.get('kafka.confluent.username'), - password: config.get('kafka.confluent.password') - }; - kafkaConfig['ssl'] = true; - } + if (this.kafkaClientId) { + kafkaConfig['clientId'] = this.kafkaClientId; + } else { + this.logger.warn('KAFKA_CLIENT_ID is undefined. Consider to define the env variable KAFKA_CLIENT_ID'); + } - this.parseTopicProperties(); + kafkaConfig['requestTimeout'] = this.requestTimeout; - this.kafkaClient = new Kafka(kafkaConfig); - this.kafkaAdmin = this.kafkaClient.admin(); - await this.kafkaAdmin.connect(); + if (useConfluent) { + kafkaConfig['sasl'] = { + mechanism: config.get('kafka.confluent.sasl.mechanism') as any, + username: config.get('kafka.confluent.username'), + password: config.get('kafka.confluent.password') + }; + kafkaConfig['ssl'] = true; + } - let partitions = 1; + this.parseTopicProperties(); - for (let i = 0; i < this.configEntries.length; i++) { - let param = this.configEntries[i]; - if (param.name === 'partitions') { - partitions = param.value; - this.configEntries.splice(i, 1); - break; - } - } + this.kafkaClient = new Kafka(kafkaConfig); + this.kafkaAdmin = this.kafkaClient.admin(); + await this.kafkaAdmin.connect(); - let topics = await this.kafkaAdmin.listTopics(); + let partitions = 1; - if (!topics.includes(requestTopic)) { - let createRequestTopicResult = await this.createTopic(requestTopic, partitions); - if (createRequestTopicResult) { - this.logger.info('Created new topic: %s', requestTopic); - } + for (let i = 0; i < this.configEntries.length; i++) { + let param = this.configEntries[i]; + if (param.name === 'partitions') { + partitions = param.value; + this.configEntries.splice(i, 1); + break; } + } - this.consumer = this.kafkaClient.consumer({groupId: 'js-executor-group'}); - this.producer = this.kafkaClient.producer({createPartitioner: Partitioners.DefaultPartitioner}); + let topics = await this.kafkaAdmin.listTopics(); - const {CRASH} = this.consumer.events; + if (!topics.includes(requestTopic)) { + let createRequestTopicResult = await this.createTopic(requestTopic, partitions); + if (createRequestTopicResult) { + this.logger.info('Created new topic: %s', requestTopic); + } + } - this.consumer.on(CRASH, async (e) => { - this.logger.error(`Got consumer CRASH event, should restart: ${e.payload.restart}`); - if (!e.payload.restart) { - this.logger.error('Going to exit due to not retryable error!'); - await this.destroy(-1); - } - }); + this.consumer = this.kafkaClient.consumer({groupId: 'js-executor-group'}); + this.producer = this.kafkaClient.producer({createPartitioner: Partitioners.DefaultPartitioner}); - const messageProcessor = new JsInvokeMessageProcessor(this); - await this.consumer.connect(); - await this.producer.connect(); - this.sendLoopWithLinger(); - await this.consumer.subscribe({topic: requestTopic}); - - await this.consumer.run({ - partitionsConsumedConcurrently: this.partitionsConsumedConcurrently, - eachMessage: async ({topic, partition, message}) => { - let headers = message.headers; - let key = message.key || new Buffer([]); - let msg = { - key: key.toString('utf8'), - data: message.value, - headers: { - data: headers - } - }; - messageProcessor.onJsInvokeMessage(msg); - }, - }); + const {CRASH} = this.consumer.events; - } catch (e: any) { - this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - this.logger.error(e.stack); - await this.destroy(-1); - } - } + this.consumer.on(CRASH, async (e) => { + this.logger.error(`Got consumer CRASH event, should restart: ${e.payload.restart}`); + if (!e.payload.restart) { + this.logger.error('Going to exit due to not retryable error!'); + await this.destroy(); + } + }); + + const messageProcessor = new JsInvokeMessageProcessor(this); + await this.consumer.connect(); + await this.producer.connect(); + this.sendLoopWithLinger(); + await this.consumer.subscribe({topic: requestTopic}); + + await this.consumer.run({ + partitionsConsumedConcurrently: this.partitionsConsumedConcurrently, + eachMessage: async ({topic, partition, message}) => { + let headers = message.headers; + let key = message.key || new Buffer([]); + let msg = { + key: key.toString('utf8'), + data: message.value, + headers: { + data: headers + } + }; + messageProcessor.onJsInvokeMessage(msg); + }, + }); +} async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise { this.logger.debug('Pending queue response, scriptId: [%s]', scriptId); @@ -232,15 +227,7 @@ export class KafkaTemplate implements IQueue { }, this.linger); } - static async build(): Promise { - const queue = new KafkaTemplate(); - await queue.init(); - return queue; - } - - - async destroy(status: number): Promise { - this.logger.info('Exiting with status: %d ...', status); + async destroy(): Promise { this.logger.info('Stopping Kafka resources...'); if (this.kafkaAdmin) { @@ -267,7 +254,6 @@ export class KafkaTemplate implements IQueue { } } this.logger.info('Kafka resources stopped.'); - process.exit(status); } private async disconnectProducer(): Promise { @@ -287,4 +273,5 @@ export class KafkaTemplate implements IQueue { } } } + } diff --git a/msa/js-executor/queue/pubSubTemplate.ts b/msa/js-executor/queue/pubSubTemplate.ts index 4e8990a105..eff35017ba 100644 --- a/msa/js-executor/queue/pubSubTemplate.ts +++ b/msa/js-executor/queue/pubSubTemplate.ts @@ -34,55 +34,50 @@ export class PubSubTemplate implements IQueue { private topics: string[] = []; private subscriptions: string[] = []; + name = 'Pub/Sub'; + constructor() { } async init() { - try { - this.pubSubClient = new PubSub({ - projectId: this.projectId, - credentials: this.credentials - }); - - this.parseQueueProperties(); + this.pubSubClient = new PubSub({ + projectId: this.projectId, + credentials: this.credentials + }); - const topicList = await this.pubSubClient.getTopics(); + this.parseQueueProperties(); - if (topicList) { - topicList[0].forEach(topic => { - this.topics.push(PubSubTemplate.getName(topic.name)); - }); - } + const topicList = await this.pubSubClient.getTopics(); - const subscriptionList = await this.pubSubClient.getSubscriptions(); + if (topicList) { + topicList[0].forEach(topic => { + this.topics.push(PubSubTemplate.getName(topic.name)); + }); + } - if (subscriptionList) { - topicList[0].forEach(sub => { - this.subscriptions.push(PubSubTemplate.getName(sub.name)); - }); - } + const subscriptionList = await this.pubSubClient.getSubscriptions(); - if (!(this.subscriptions.includes(this.requestTopic) && this.topics.includes(this.requestTopic))) { - await this.createTopic(this.requestTopic); - await this.createSubscription(this.requestTopic); - } + if (subscriptionList) { + topicList[0].forEach(sub => { + this.subscriptions.push(PubSubTemplate.getName(sub.name)); + }); + } - const subscription = this.pubSubClient.subscription(this.requestTopic); + if (!(this.subscriptions.includes(this.requestTopic) && this.topics.includes(this.requestTopic))) { + await this.createTopic(this.requestTopic); + await this.createSubscription(this.requestTopic); + } - const messageProcessor = new JsInvokeMessageProcessor(this); + const subscription = this.pubSubClient.subscription(this.requestTopic); - const messageHandler = (message: Message) => { - messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8'))); - message.ack(); - }; + const messageProcessor = new JsInvokeMessageProcessor(this); - subscription.on('message', messageHandler); + const messageHandler = (message: Message) => { + messageProcessor.onJsInvokeMessage(JSON.parse(message.data.toString('utf8'))); + message.ack(); + }; - } catch (e: any) { - this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - this.logger.error(e.stack); - await this.destroy(-1); - } + subscription.on('message', messageHandler); } async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise { @@ -146,14 +141,7 @@ export class PubSubTemplate implements IQueue { } } - static async build(): Promise { - const queue = new PubSubTemplate(); - await queue.init(); - return queue; - } - - async destroy(status: number): Promise { - this.logger.info('Exiting with status: %d ...', status); + async destroy(): Promise { this.logger.info('Stopping Pub/Sub resources...'); if (this.pubSubClient) { this.logger.info('Stopping Pub/Sub client...'); @@ -168,7 +156,6 @@ export class PubSubTemplate implements IQueue { } } this.logger.info('Pub/Sub resources stopped.'); - process.exit(status); } } diff --git a/msa/js-executor/queue/queue.models.ts b/msa/js-executor/queue/queue.models.ts index 59ec68896d..a86dc8fd1d 100644 --- a/msa/js-executor/queue/queue.models.ts +++ b/msa/js-executor/queue/queue.models.ts @@ -15,7 +15,8 @@ /// export interface IQueue { + name: string; init(): Promise; send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise; - destroy(status: number): Promise; + destroy(): Promise; } diff --git a/msa/js-executor/queue/rabbitmqTemplate.ts b/msa/js-executor/queue/rabbitmqTemplate.ts index 372024a4f3..9369f170f5 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.ts +++ b/msa/js-executor/queue/rabbitmqTemplate.ts @@ -44,39 +44,35 @@ export class RabbitMqTemplate implements IQueue { private stopped = false; private topics: string[] = []; + name = 'RabbitMQ'; + constructor() { } async init(): Promise { - try { - const url = `amqp://${this.username}:${this.password}@${this.host}:${this.port}${this.vhost}`; - this.connection = await amqp.connect(url); - this.channel = await this.connection.createConfirmChannel(); + const url = `amqp://${this.username}:${this.password}@${this.host}:${this.port}${this.vhost}`; + this.connection = await amqp.connect(url); + this.channel = await this.connection.createConfirmChannel(); - this.parseQueueProperties(); + this.parseQueueProperties(); - await this.createQueue(this.requestTopic); + await this.createQueue(this.requestTopic); - const messageProcessor = new JsInvokeMessageProcessor(this); + const messageProcessor = new JsInvokeMessageProcessor(this); - while (!this.stopped) { - let pollStartTs = new Date().getTime(); - let message = await this.channel.get(this.requestTopic); + while (!this.stopped) { + let pollStartTs = new Date().getTime(); + let message = await this.channel.get(this.requestTopic); - if (message) { - messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); - this.channel.ack(message); - } else { - let pollDuration = new Date().getTime() - pollStartTs; - if (pollDuration < this.pollInterval) { - await sleep(this.pollInterval - pollDuration); - } + if (message) { + messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); + this.channel.ack(message); + } else { + let pollDuration = new Date().getTime() - pollStartTs; + if (pollDuration < this.pollInterval) { + await sleep(this.pollInterval - pollDuration); } } - } catch (e: any) { - this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - this.logger.error(e.stack); - await this.destroy(-1); } } @@ -112,14 +108,7 @@ export class RabbitMqTemplate implements IQueue { return this.channel.assertQueue(topic, this.queueOptions); } - static async build(): Promise { - const queue = new RabbitMqTemplate(); - await queue.init(); - return queue; - } - - async destroy(status: number) { - this.logger.info('Exiting with status: %d ...', status); + async destroy() { this.logger.info('Stopping RabbitMQ resources...'); if (this.channel) { @@ -144,7 +133,6 @@ export class RabbitMqTemplate implements IQueue { } } this.logger.info('RabbitMQ resources stopped.') - process.exit(status); } } diff --git a/msa/js-executor/queue/serviceBusTemplate.ts b/msa/js-executor/queue/serviceBusTemplate.ts index b2750672e5..76d87e8068 100644 --- a/msa/js-executor/queue/serviceBusTemplate.ts +++ b/msa/js-executor/queue/serviceBusTemplate.ts @@ -44,46 +44,42 @@ export class ServiceBusTemplate implements IQueue { private receiver: ServiceBusReceiver; private senderMap = new Map(); + name = 'Azure Service Bus'; + constructor() { } async init() { - try { - const connectionString = `Endpoint=sb://${this.namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${this.sasKeyName};SharedAccessKey=${this.sasKey}`; - this.sbClient = new ServiceBusClient(connectionString) - this.serviceBusService = new ServiceBusAdministrationClient(connectionString); + const connectionString = `Endpoint=sb://${this.namespaceName}.servicebus.windows.net/;SharedAccessKeyName=${this.sasKeyName};SharedAccessKey=${this.sasKey}`; + this.sbClient = new ServiceBusClient(connectionString) + this.serviceBusService = new ServiceBusAdministrationClient(connectionString); - this.parseQueueProperties(); + this.parseQueueProperties(); - const listQueues = await this.serviceBusService.listQueues(); - for await (const queue of listQueues) { - this.queues.push(queue.name); - } - - if (!this.queues.includes(this.requestTopic)) { - await this.createQueueIfNotExist(this.requestTopic); - this.queues.push(this.requestTopic); - } + const listQueues = await this.serviceBusService.listQueues(); + for await (const queue of listQueues) { + this.queues.push(queue.name); + } - this.receiver = this.sbClient.createReceiver(this.requestTopic, {receiveMode: 'peekLock'}); - - const messageProcessor = new JsInvokeMessageProcessor(this); - - const messageHandler = async (message: ServiceBusReceivedMessage) => { - if (message) { - messageProcessor.onJsInvokeMessage(message.body); - await this.receiver.completeMessage(message); - } - }; - const errorHandler = async (error: ProcessErrorArgs) => { - this.logger.error('Failed to receive message from queue.', error); - }; - this.receiver.subscribe({processMessage: messageHandler, processError: errorHandler}) - } catch (e: any) { - this.logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); - this.logger.error(e.stack); - await this.destroy(-1); + if (!this.queues.includes(this.requestTopic)) { + await this.createQueueIfNotExist(this.requestTopic); + this.queues.push(this.requestTopic); } + + this.receiver = this.sbClient.createReceiver(this.requestTopic, {receiveMode: 'peekLock'}); + + const messageProcessor = new JsInvokeMessageProcessor(this); + + const messageHandler = async (message: ServiceBusReceivedMessage) => { + if (message) { + messageProcessor.onJsInvokeMessage(message.body); + await this.receiver.completeMessage(message); + } + }; + const errorHandler = async (error: ProcessErrorArgs) => { + this.logger.error('Failed to receive message from queue.', error); + }; + this.receiver.subscribe({processMessage: messageHandler, processError: errorHandler}) } async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise { @@ -133,14 +129,7 @@ export class ServiceBusTemplate implements IQueue { } } - static async build(): Promise { - const queue = new ServiceBusTemplate(); - await queue.init(); - return queue; - } - - async destroy(status: number) { - this.logger.info('Exiting with status: %d ...', status); + async destroy() { this.logger.info('Stopping Azure Service Bus resources...') if (this.receiver) { this.logger.info('Stopping Service Bus Receiver...'); @@ -181,6 +170,5 @@ export class ServiceBusTemplate implements IQueue { } } this.logger.info('Azure Service Bus resources stopped.') - process.exit(status); } } diff --git a/msa/js-executor/server.ts b/msa/js-executor/server.ts index 708a87fec9..446f9d34aa 100644 --- a/msa/js-executor/server.ts +++ b/msa/js-executor/server.ts @@ -30,63 +30,66 @@ logger.info('===CONFIG BEGIN==='); logger.info(JSON.stringify(config, null, 4)); logger.info('===CONFIG END==='); -const serviceType = config.get('queue_type'); +const serviceType: string = config.get('queue_type'); const httpPort = Number(config.get('http_port')); let queues: IQueue | null; let httpServer: HttpServer | null; (async () => { logger.info('Starting ThingsBoard JavaScript Executor Microservice...'); + try { + queues = await createQueue(serviceType); + logger.info(`Starting ${queues.name} template...`); + await queues.init(); + logger.info(`${queues.name} template started.`); + httpServer = new HttpServer(httpPort); + } catch (e: any) { + logger.error('Failed to start ThingsBoard JavaScript Executor Microservice: %s', e.message); + logger.error(e.stack); + await exit(-1); + } + +})(); + +async function createQueue(serviceType: string): Promise { switch (serviceType) { case 'kafka': - logger.info('Starting Kafka template...'); - queues = await KafkaTemplate.build(); - logger.info('Kafka template started.'); - break; + return new KafkaTemplate(); case 'pubsub': - logger.info('Starting Pub/Sub template...') - queues = await PubSubTemplate.build(); - logger.info('Pub/Sub template started.') - break; + return new PubSubTemplate(); case 'aws-sqs': - logger.info('Starting AWS SQS template...') - queues = await AwsSqsTemplate.build(); - logger.info('AWS SQS template started.') - break; + return new AwsSqsTemplate(); case 'rabbitmq': - logger.info('Starting RabbitMQ template...') - queues = await RabbitMqTemplate.build(); - logger.info('RabbitMQ template started.') - break; + return new RabbitMqTemplate(); case 'service-bus': - logger.info('Starting Azure Service Bus template...') - queues = await ServiceBusTemplate.build(); - logger.info('Azure Service Bus template started.') - break; + return new ServiceBusTemplate(); default: - logger.error('Unknown service type: ', serviceType); - process.exit(-1); + throw new Error('Unknown service type: ' + serviceType); } - - httpServer = new HttpServer(httpPort); -})(); +} [`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach((eventType) => { process.on(eventType, async () => { logger.info(`${eventType} signal received`); - if (httpServer) { - const _httpServer = httpServer; - httpServer = null; - await _httpServer.stop(); - } - if (queues) { - const _queues = queues; - queues = null; - await _queues.destroy(0); - } + await exit(0); }) }) process.on('exit', (code: number) => { - logger.info(`JavaScript Executor Microservice has been stopped. Exit code: ${code}.`); + logger.info(`ThingsBoard JavaScript Executor Microservice has been stopped. Exit code: ${code}.`); }); + +async function exit(status: number) { + logger.info('Exiting with status: %d ...', status); + if (httpServer) { + const _httpServer = httpServer; + httpServer = null; + await _httpServer.stop(); + } + if (queues) { + const _queues = queues; + queues = null; + await _queues.destroy(); + } + process.exit(status); +} diff --git a/msa/js-executor/yarn.lock b/msa/js-executor/yarn.lock index ed6e7406f9..eb6443e53d 100644 --- a/msa/js-executor/yarn.lock +++ b/msa/js-executor/yarn.lock @@ -2670,10 +2670,10 @@ jws@^4.0.0: jwa "^2.0.0" safe-buffer "^5.0.1" -kafkajs@^2.0.2: - version "2.0.2" - resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.0.2.tgz#cdfc8f57aa4fd69f6d9ca1cce4ee89bbc2a3a1f9" - integrity sha512-g6CM3fAenofOjR1bfOAqeZUEaSGhNtBscNokybSdW1rmIKYNwBPC9xQzwulFJm36u/xcxXUiCl/L/qfslapihA== +kafkajs@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/kafkajs/-/kafkajs-2.1.0.tgz#32ede4e8080cc75586c5e4406eeb582fa73f7b1e" + integrity sha512-6IYiOdGWvFPbSbVB+AV3feT+A7vzw5sXm7Ze4QTfP7FRNdY8pGcpiNPvD2lfgYFD8Dm9KbMgBgTt2mf8KaIkzw== keyv@^3.0.0: version "3.1.0" From a6a19cf95590ce74038d232bdc1c2ebefc7775e2 Mon Sep 17 00:00:00 2001 From: nickAS21 Date: Thu, 7 Jul 2022 19:53:45 +0300 Subject: [PATCH 18/28] refactoring: tests TenantProfile --- .../controller/AbstractNotifyEntityTest.java | 4 +- .../controller/BaseTenantControllerTest.java | 14 +-- .../BaseTenantProfileControllerTest.java | 112 +++++++++++++++--- 3 files changed, 106 insertions(+), 24 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java index 87d41b8e2f..c75425c870 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractNotifyEntityTest.java @@ -381,8 +381,8 @@ public abstract class AbstractNotifyEntityTest extends AbstractWebTest { } protected void testBroadcastEntityStateChangeEventTime(EntityId entityId, TenantId tenantId, int cntTime) { - ArgumentMatcher matcherTenantIdId = cntTime == 1 ? argument -> argument.equals(tenantId) : - argument -> argument.getClass().equals(TenantId.class); + ArgumentMatcher matcherTenantIdId = cntTime > 1 || tenantId == null ? argument -> argument.getClass().equals(TenantId.class) : + argument -> argument.equals(tenantId) ; Mockito.verify(tbClusterService, times(cntTime)).broadcastEntityStateChangeEvent(Mockito.argThat(matcherTenantIdId), Mockito.any(entityId.getClass()), Mockito.any(ComponentLifecycleEvent.class)); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java index a59d8612d1..1840f1f8a0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantControllerTest.java @@ -93,7 +93,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); - Mockito.reset(tbClusterService, auditLogService); + Mockito.reset(tbClusterService); Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); Assert.assertNotNull(savedTenant); @@ -122,7 +122,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { Tenant tenant = new Tenant(); tenant.setTitle(RandomStringUtils.randomAlphanumeric(300)); - Mockito.reset(tbClusterService, auditLogService); + Mockito.reset(tbClusterService); doPost("/api/tenant", tenant) .andExpect(status().isBadRequest()) @@ -161,7 +161,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { public void testSaveTenantWithEmptyTitle() throws Exception { loginSysAdmin(); - Mockito.reset(tbClusterService, auditLogService); + Mockito.reset(tbClusterService); Tenant tenant = new Tenant(); doPost("/api/tenant", tenant) @@ -175,7 +175,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { public void testSaveTenantWithInvalidEmail() throws Exception { loginSysAdmin(); - Mockito.reset(tbClusterService, auditLogService); + Mockito.reset(tbClusterService); Tenant tenant = new Tenant(); tenant.setTitle("My tenant"); @@ -212,7 +212,7 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { Assert.assertEquals(1, pageData.getData().size()); tenants.addAll(pageData.getData()); - Mockito.reset(tbClusterService, auditLogService); + Mockito.reset(tbClusterService); int cntEntity = 56; List> createFutures = new ArrayList<>(56); @@ -578,13 +578,13 @@ public abstract class BaseTenantControllerTest extends AbstractControllerTest { } TenantId tenantId = cntTime == 1 ? tenant.getId() : (TenantId) createEntityId_NULL_UUID(tenant); testBroadcastEntityStateChangeEventTime(tenantId, tenantId, cntTime); - Mockito.reset(tbClusterService, auditLogService); + Mockito.reset(tbClusterService); } private void testBroadcastEntityStateChangeEventNeverTenant() { Mockito.verify(tbClusterService, never()).onTenantChange(Mockito.any(Tenant.class), Mockito.isNull()); testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); - Mockito.reset(tbClusterService, auditLogService); + Mockito.reset(tbClusterService); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java index 1867577c23..33eca6a617 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseTenantProfileControllerTest.java @@ -17,24 +17,24 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.core.type.TypeReference; import org.apache.commons.lang3.RandomStringUtils; -import org.junit.After; import org.junit.Assert; import org.junit.Test; -import org.springframework.beans.factory.annotation.Autowired; +import org.mockito.ArgumentMatcher; +import org.mockito.Mockito; import org.thingsboard.server.common.data.EntityInfo; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantProfileId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.common.data.queue.ProcessingStrategy; import org.thingsboard.server.common.data.queue.ProcessingStrategyType; import org.thingsboard.server.common.data.queue.SubmitStrategy; import org.thingsboard.server.common.data.queue.SubmitStrategyType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; -import org.thingsboard.server.dao.tenant.TenantProfileService; import java.util.ArrayList; import java.util.Collections; @@ -42,6 +42,8 @@ import java.util.List; import java.util.stream.Collectors; import static org.hamcrest.Matchers.containsString; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; public abstract class BaseTenantProfileControllerTest extends AbstractControllerTest { @@ -52,6 +54,9 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController @Test public void testSaveTenantProfile() throws Exception { loginSysAdmin(); + + Mockito.reset(tbClusterService); + TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); Assert.assertNotNull(savedTenantProfile); @@ -64,17 +69,28 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController Assert.assertEquals(tenantProfile.isIsolatedTbCore(), savedTenantProfile.isIsolatedTbCore()); Assert.assertEquals(tenantProfile.isIsolatedTbRuleEngine(), savedTenantProfile.isIsolatedTbRuleEngine()); + testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(savedTenantProfile, ComponentLifecycleEvent.CREATED, 1); + savedTenantProfile.setName("New tenant profile"); doPost("/api/tenantProfile", savedTenantProfile, TenantProfile.class); TenantProfile foundTenantProfile = doGet("/api/tenantProfile/"+savedTenantProfile.getId().getId().toString(), TenantProfile.class); Assert.assertEquals(foundTenantProfile.getName(), savedTenantProfile.getName()); + + testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(savedTenantProfile, ComponentLifecycleEvent.UPDATED, 1); } @Test public void testSaveTenantProfileWithViolationOfLengthValidation() throws Exception { loginSysAdmin(); + + Mockito.reset(tbClusterService); + TenantProfile tenantProfile = this.createTenantProfile(RandomStringUtils.randomAlphabetic(300)); - doPost("/api/tenantProfile", tenantProfile).andExpect(statusReason(containsString("length of name must be equal or less than 255"))); + doPost("/api/tenantProfile", tenantProfile) + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString(msgErrorFieldLength("name")))); + + testBroadcastEntityStateChangeEventNeverTenantProfile(); } @Test @@ -122,9 +138,15 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController @Test public void testSaveTenantProfileWithEmptyName() throws Exception { loginSysAdmin(); + + Mockito.reset(tbClusterService); + TenantProfile tenantProfile = new TenantProfile(); doPost("/api/tenantProfile", tenantProfile).andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Tenant profile name should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Tenant profile name " + msgErrorShouldBeSpecified))); + + testBroadcastEntityStateChangeEventNeverTenantProfile(); } @Test @@ -132,9 +154,15 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); doPost("/api/tenantProfile", tenantProfile).andExpect(status().isOk()); + + Mockito.reset(tbClusterService); + TenantProfile tenantProfile2 = this.createTenantProfile("Tenant Profile"); - doPost("/api/tenantProfile", tenantProfile2).andExpect(status().isBadRequest()) + doPost("/api/tenantProfile", tenantProfile2) + .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString("Tenant profile with such name already exists"))); + + testBroadcastEntityStateChangeEventNeverTenantProfile(); } @Test @@ -144,8 +172,14 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); savedTenantProfile.setIsolatedTbRuleEngine(true); addMainQueueConfig(savedTenantProfile); - doPost("/api/tenantProfile", savedTenantProfile).andExpect(status().isBadRequest()) + + Mockito.reset(tbClusterService); + + doPost("/api/tenantProfile", savedTenantProfile) + .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString("Can't update isolatedTbRuleEngine property"))); + + testBroadcastEntityStateChangeEventNeverTenantProfile(); } @Test @@ -153,9 +187,15 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController loginSysAdmin(); TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + + Mockito.reset(tbClusterService); + savedTenantProfile.setIsolatedTbCore(true); - doPost("/api/tenantProfile", savedTenantProfile).andExpect(status().isBadRequest()) + doPost("/api/tenantProfile", savedTenantProfile) + .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString("Can't update isolatedTbCore property"))); + + testBroadcastEntityStateChangeEventNeverTenantProfile(); } @Test @@ -169,10 +209,14 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController tenant.setTenantProfileId(savedTenantProfile.getId()); Tenant savedTenant = doPost("/api/tenant", tenant, Tenant.class); + Mockito.reset(tbClusterService); + doDelete("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString()) .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString("The tenant profile referenced by the tenants cannot be deleted"))); + testBroadcastEntityStateChangeEventNeverTenantProfile(); + doDelete("/api/tenant/"+savedTenant.getId().getId().toString()) .andExpect(status().isOk()); } @@ -183,11 +227,16 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"); TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + Mockito.reset(tbClusterService); + doDelete("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString()) .andExpect(status().isOk()); + testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(savedTenantProfile, ComponentLifecycleEvent.DELETED, 1); + doGet("/api/tenantProfile/" + savedTenantProfile.getId().getId().toString()) - .andExpect(status().isNotFound()); + .andExpect(status().isNotFound()) + .andExpect(statusReason(containsString(msgErrorNoFound("Tenant profile", savedTenantProfile.getId().getId().toString())))); } @Test @@ -196,21 +245,26 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController List tenantProfiles = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference>(){}, pageLink); + new TypeReference<>(){}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); tenantProfiles.addAll(pageData.getData()); + Mockito.reset(tbClusterService); + + int cntEntity = 28; for (int i=0;i<28;i++) { TenantProfile tenantProfile = this.createTenantProfile("Tenant Profile"+i); tenantProfiles.add(doPost("/api/tenantProfile", tenantProfile, TenantProfile.class)); } + testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(new TenantProfile(), ComponentLifecycleEvent.CREATED, cntEntity); + List loadedTenantProfiles = new ArrayList<>(); pageLink = new PageLink(17); do { pageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference>(){}, pageLink); + new TypeReference<>(){}, pageLink); loadedTenantProfiles.addAll(pageData.getData()); if (pageData.hasNext()) { pageLink = pageLink.nextPageLink(); @@ -222,6 +276,8 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController Assert.assertEquals(tenantProfiles, loadedTenantProfiles); + Mockito.reset(tbClusterService); + for (TenantProfile tenantProfile : loadedTenantProfiles) { if (!tenantProfile.isDefault()) { doDelete("/api/tenantProfile/" + tenantProfile.getId().getId().toString()) @@ -234,6 +290,8 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController new TypeReference>(){}, pageLink); Assert.assertFalse(pageData.hasNext()); Assert.assertEquals(1, pageData.getTotalElements()); + + testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(new TenantProfile(), ComponentLifecycleEvent.DELETED, cntEntity); } @Test @@ -242,7 +300,7 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController List tenantProfiles = new ArrayList<>(); PageLink pageLink = new PageLink(17); PageData tenantProfilePageData = doGetTypedWithPageLink("/api/tenantProfiles?", - new TypeReference>(){}, pageLink); + new TypeReference<>(){}, pageLink); Assert.assertFalse(tenantProfilePageData.hasNext()); Assert.assertEquals(1, tenantProfilePageData.getTotalElements()); tenantProfiles.addAll(tenantProfilePageData.getData()); @@ -322,4 +380,28 @@ public abstract class BaseTenantProfileControllerTest extends AbstractController profileData.setQueueConfiguration(Collections.singletonList(mainQueueConfiguration)); tenantProfile.setProfileData(profileData); } + + + private void testBroadcastEntityStateChangeEventTimeManyTimeTenantProfile(TenantProfile tenantProfile, ComponentLifecycleEvent event, int cntTime) { + ArgumentMatcher matcherTenantProfile = cntTime == 1 ? argument -> argument.equals(tenantProfile) : + argument -> argument.getClass().equals(TenantProfile.class); + if (ComponentLifecycleEvent.DELETED.equals(event)) { + Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileDelete(Mockito.argThat( matcherTenantProfile), + Mockito.isNull()); + testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); + } else { + Mockito.verify(tbClusterService, times( cntTime)).onTenantProfileChange(Mockito.argThat(matcherTenantProfile), + Mockito.isNull()); + TenantProfileId tenantProfileIdId = cntTime == 1 ? tenantProfile.getId() : (TenantProfileId) createEntityId_NULL_UUID(tenantProfile); + testBroadcastEntityStateChangeEventTime(tenantProfileIdId, null, cntTime); + } + Mockito.reset(tbClusterService); + } + + private void testBroadcastEntityStateChangeEventNeverTenantProfile() { + Mockito.verify(tbClusterService, never()).onTenantProfileChange(Mockito.any(TenantProfile.class), + Mockito.isNull()); + testBroadcastEntityStateChangeEventNever(createEntityId_NULL_UUID(new Tenant())); + Mockito.reset(tbClusterService, auditLogService); + } } From 5f183f5f38996b8f42fd33f85ef72a3683c53e55 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 7 Jul 2022 20:29:28 +0300 Subject: [PATCH 19/28] Improve web-ui and js-executor microservices gracefull shutdown --- msa/js-executor/server.ts | 2 +- msa/web-ui/server.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/msa/js-executor/server.ts b/msa/js-executor/server.ts index 446f9d34aa..035bf77183 100644 --- a/msa/js-executor/server.ts +++ b/msa/js-executor/server.ts @@ -69,7 +69,7 @@ async function createQueue(serviceType: string): Promise { } [`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach((eventType) => { - process.on(eventType, async () => { + process.once(eventType, async () => { logger.info(`${eventType} signal received`); await exit(0); }) diff --git a/msa/web-ui/server.ts b/msa/web-ui/server.ts index 2aafe879d1..5ba56add1a 100644 --- a/msa/web-ui/server.ts +++ b/msa/web-ui/server.ts @@ -123,7 +123,7 @@ let connections: Socket[] = []; })(); [`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach((eventType) => { - process.on(eventType, async () => { + process.once(eventType, async () => { logger.info(`${eventType} signal received`); await exit(0); }); From ca0ceeb3e124695660deb53eef6f93873006ef74 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 7 Jul 2022 19:32:29 +0200 Subject: [PATCH 20/28] fixed device & device-profile validation --- .../server/controller/AbstractWebTest.java | 2 +- .../controller/BaseDeviceControllerTest.java | 67 ++++++++- .../BaseDeviceProfileControllerTest.java | 141 +++++++++++++++++- .../server/dao/device/DeviceServiceImpl.java | 4 + .../AbstractHasOtaPackageValidator.java | 66 ++++++++ .../validator/DeviceDataValidator.java | 41 +---- .../validator/DeviceProfileDataValidator.java | 53 ++----- 7 files changed, 294 insertions(+), 80 deletions(-) create mode 100644 dao/src/main/java/org/thingsboard/server/dao/service/validator/AbstractHasOtaPackageValidator.java diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 329bad55a5..677303cc15 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -285,7 +285,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected void loginDifferentTenant() throws Exception { if (savedDifferentTenant != null) { - login(savedDifferentTenant.getEmail(), TENANT_ADMIN_PASSWORD); + login(DIFFERENT_TENANT_ADMIN_EMAIL, DIFFERENT_TENANT_ADMIN_PASSWORD); } else { loginSysAdmin(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java index 4c4f44aea6..8ea89960d7 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceControllerTest.java @@ -30,7 +30,10 @@ import org.mockito.Mockito; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntitySubtype; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; @@ -57,6 +60,8 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; +import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; public abstract class BaseDeviceControllerTest extends AbstractControllerTest { @@ -204,7 +209,7 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { String savedDeviceIdStr = savedDevice.getId().getId().toString(); doPost("/api/device", savedDevice) - .andExpect( status().isNotFound()) + .andExpect(status().isNotFound()) .andExpect(statusReason(containsString(msgErrorNoFound("Device", savedDeviceIdStr)))); testNotifyEntityNever(savedDevice.getId(), savedDevice); @@ -222,6 +227,66 @@ public abstract class BaseDeviceControllerTest extends AbstractControllerTest { deleteDifferentTenant(); } + @Test + public void testSaveDeviceWithProfileFromDifferentTenant() throws Exception { + loginDifferentTenant(); + DeviceProfile differentProfile = createDeviceProfile("Different profile"); + differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + + loginTenantAdmin(); + Device device = new Device(); + device.setName("My device"); + device.setDeviceProfileId(differentProfile.getId()); + doPost("/api/device", device).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Device can`t be referencing to device profile from different tenant!"))); + } + + @Test + public void testSaveDeviceWithFirmwareFromDifferentTenant() throws Exception { + loginDifferentTenant(); + DeviceProfile differentProfile = createDeviceProfile("Different profile"); + differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); + firmwareInfo.setDeviceProfileId(differentProfile.getId()); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle("title"); + firmwareInfo.setVersion("1.0"); + firmwareInfo.setUrl("test.url"); + firmwareInfo.setUsesUrl(true); + OtaPackageInfo savedFw = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); + + loginTenantAdmin(); + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + device.setFirmwareId(savedFw.getId()); + doPost("/api/device", device).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Can't assign firmware from different tenant!"))); + } + + @Test + public void testSaveDeviceWithSoftwareFromDifferentTenant() throws Exception { + loginDifferentTenant(); + DeviceProfile differentProfile = createDeviceProfile("Different profile"); + differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + SaveOtaPackageInfoRequest softwareInfo = new SaveOtaPackageInfoRequest(); + softwareInfo.setDeviceProfileId(differentProfile.getId()); + softwareInfo.setType(SOFTWARE); + softwareInfo.setTitle("title"); + softwareInfo.setVersion("1.0"); + softwareInfo.setUrl("test.url"); + softwareInfo.setUsesUrl(true); + OtaPackageInfo savedSw = doPost("/api/otaPackage", softwareInfo, OtaPackageInfo.class); + + loginTenantAdmin(); + Device device = new Device(); + device.setName("My device"); + device.setType("default"); + device.setSoftwareId(savedSw.getId()); + doPost("/api/device", device).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Can't assign software from different tenant!"))); + } + @Test public void testFindDeviceById() throws Exception { Device device = new Device(); diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java index 1fb5316a17..d0c4a1d7ce 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseDeviceProfileControllerTest.java @@ -29,13 +29,17 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; @@ -45,7 +49,15 @@ import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadCo import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.queue.ProcessingStrategy; +import org.thingsboard.server.common.data.queue.ProcessingStrategyType; +import org.thingsboard.server.common.data.queue.Queue; +import org.thingsboard.server.common.data.queue.SubmitStrategy; +import org.thingsboard.server.common.data.queue.SubmitStrategyType; +import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.ArrayList; @@ -59,6 +71,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; +import static org.thingsboard.server.common.data.ota.OtaPackageType.SOFTWARE; public abstract class BaseDeviceProfileControllerTest extends AbstractControllerTest { @@ -345,6 +359,129 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController testNotifyEntityNever(savedDeviceProfile.getId(), savedDeviceProfile); } + @Test + public void testSaveDeviceProfileWithRuleChainFromDifferentTenant() throws Exception { + loginDifferentTenant(); + RuleChain ruleChain = new RuleChain(); + ruleChain.setName("Different rule chain"); + RuleChain savedRuleChain = doPost("/api/ruleChain", ruleChain, RuleChain.class); + + loginTenantAdmin(); + + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + deviceProfile.setDefaultRuleChainId(savedRuleChain.getId()); + doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Can't assign rule chain from different tenant!"))); + } + + @Test + public void testSaveDeviceProfileWithDashboardFromDifferentTenant() throws Exception { + loginDifferentTenant(); + Dashboard dashboard = new Dashboard(); + dashboard.setTitle("Different dashboard"); + Dashboard savedDashboard = doPost("/api/dashboard", dashboard, Dashboard.class); + + loginTenantAdmin(); + + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + deviceProfile.setDefaultDashboardId(savedDashboard.getId()); + doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Can't assign dashboard from different tenant!"))); + } + + @Test + public void testSaveDeviceProfileWithQueueFromDifferentTenant() throws Exception { + loginDifferentTenant(); + loginSysAdmin(); + TenantProfile tenantProfile = new TenantProfile(); + tenantProfile.setDefault(false); + tenantProfile.setName("Isolated TB Rule Engine"); + tenantProfile.setDescription("Isolated TB Rule Engine tenant profile"); + tenantProfile.setIsolatedTbCore(false); + tenantProfile.setIsolatedTbRuleEngine(true); + + TenantProfileQueueConfiguration mainQueueConfiguration = new TenantProfileQueueConfiguration(); + mainQueueConfiguration.setName("Main"); + mainQueueConfiguration.setTopic("tb_rule_engine.main"); + mainQueueConfiguration.setPollInterval(25); + mainQueueConfiguration.setPartitions(10); + mainQueueConfiguration.setConsumerPerPartition(true); + mainQueueConfiguration.setPackProcessingTimeout(2000); + SubmitStrategy mainQueueSubmitStrategy = new SubmitStrategy(); + mainQueueSubmitStrategy.setType(SubmitStrategyType.BURST); + mainQueueSubmitStrategy.setBatchSize(1000); + mainQueueConfiguration.setSubmitStrategy(mainQueueSubmitStrategy); + ProcessingStrategy mainQueueProcessingStrategy = new ProcessingStrategy(); + mainQueueProcessingStrategy.setType(ProcessingStrategyType.SKIP_ALL_FAILURES); + mainQueueProcessingStrategy.setRetries(3); + mainQueueProcessingStrategy.setFailurePercentage(0); + mainQueueProcessingStrategy.setPauseBetweenRetries(3); + mainQueueProcessingStrategy.setMaxPauseBetweenRetries(3); + mainQueueConfiguration.setProcessingStrategy(mainQueueProcessingStrategy); + TenantProfileData profileData = tenantProfile.getProfileData(); + profileData.setQueueConfiguration(Collections.singletonList(mainQueueConfiguration)); + tenantProfile.setProfileData(profileData); + TenantProfile savedTenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); + savedDifferentTenant.setTenantProfileId(savedTenantProfile.getId()); + savedDifferentTenant = doPost("/api/tenant", savedDifferentTenant, Tenant.class); + loginDifferentTenant(); + PageLink pageLink = new PageLink(1); + PageData pageData = doGetTypedWithPageLink("/api/queues?serviceType=TB_RULE_ENGINE&", + new TypeReference<>() {}, pageLink); + Queue differentQueue = pageData.getData().get(0); + + loginTenantAdmin(); + + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + deviceProfile.setDefaultQueueId(differentQueue.getId()); + doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Can't assign queue from different tenant!"))); + } + + @Test + public void testSaveDeviceProfileWithFirmwareFromDifferentTenant() throws Exception { + loginDifferentTenant(); + DeviceProfile differentProfile = createDeviceProfile("Different profile"); + differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + SaveOtaPackageInfoRequest firmwareInfo = new SaveOtaPackageInfoRequest(); + firmwareInfo.setDeviceProfileId(differentProfile.getId()); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle("title"); + firmwareInfo.setVersion("1.0"); + firmwareInfo.setUrl("test.url"); + firmwareInfo.setUsesUrl(true); + OtaPackageInfo savedFw = doPost("/api/otaPackage", firmwareInfo, OtaPackageInfo.class); + + loginTenantAdmin(); + + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + deviceProfile.setFirmwareId(savedFw.getId()); + doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Can't assign firmware from different tenant!"))); + } + + @Test + public void testSaveDeviceProfileWithSoftwareFromDifferentTenant() throws Exception { + loginDifferentTenant(); + DeviceProfile differentProfile = createDeviceProfile("Different profile"); + differentProfile = doPost("/api/deviceProfile", differentProfile, DeviceProfile.class); + SaveOtaPackageInfoRequest softwareInfo = new SaveOtaPackageInfoRequest(); + softwareInfo.setDeviceProfileId(differentProfile.getId()); + softwareInfo.setType(SOFTWARE); + softwareInfo.setTitle("title"); + softwareInfo.setVersion("1.0"); + softwareInfo.setUrl("test.url"); + softwareInfo.setUsesUrl(true); + OtaPackageInfo savedSw = doPost("/api/otaPackage", softwareInfo, OtaPackageInfo.class); + + loginTenantAdmin(); + + DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); + deviceProfile.setSoftwareId(savedSw.getId()); + doPost("/api/deviceProfile", deviceProfile).andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Can't assign software from different tenant!"))); + } + @Test public void testDeleteDeviceProfile() throws Exception { DeviceProfile deviceProfile = this.createDeviceProfile("Device Profile"); @@ -1009,7 +1146,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(errorMsg))); - testNotifyEntityEqualsOneTimeServiceNeverError(deviceProfile,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(deviceProfile, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(errorMsg)); } @@ -1024,7 +1161,7 @@ public abstract class BaseDeviceProfileControllerTest extends AbstractController .andExpect(status().isBadRequest()) .andExpect(statusReason(containsString(errorMsg))); - testNotifyEntityEqualsOneTimeServiceNeverError(deviceProfile,savedTenant.getId(), + testNotifyEntityEqualsOneTimeServiceNeverError(deviceProfile, savedTenant.getId(), tenantAdmin.getId(), tenantAdmin.getEmail(), ActionType.ADDED, new DataValidationException(errorMsg)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java index 9f32939ad2..a1ab278ae3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceServiceImpl.java @@ -226,6 +226,9 @@ public class DeviceServiceImpl extends AbstractCachedEntityService> extends DataValidator { + + @Autowired + private OtaPackageService otaPackageService; + + protected void validateOtaPackage(TenantId tenantId, T entity, DeviceProfileId deviceProfileId) { + if (entity.getFirmwareId() != null) { + OtaPackage firmware = otaPackageService.findOtaPackageById(tenantId, entity.getFirmwareId()); + validateOtaPackage(tenantId, OtaPackageType.FIRMWARE, deviceProfileId, firmware); + } + if (entity.getSoftwareId() != null) { + OtaPackage software = otaPackageService.findOtaPackageById(tenantId, entity.getSoftwareId()); + validateOtaPackage(tenantId, OtaPackageType.SOFTWARE, deviceProfileId, software); + } + } + + private void validateOtaPackage(TenantId tenantId, OtaPackageType type, DeviceProfileId deviceProfileId, OtaPackage otaPackage) { + if (otaPackage == null) { + throw new DataValidationException(prepareMsg("Can't assign non-existent %s!", type)); + } + if (!otaPackage.getTenantId().equals(tenantId)) { + throw new DataValidationException(prepareMsg("Can't assign %s from different tenant!", type)); + } + if (!otaPackage.getType().equals(type)) { + throw new DataValidationException(prepareMsg("Can't assign %s with type: " + otaPackage.getType(), type)); + } + if (otaPackage.getData() == null && !otaPackage.hasUrl()) { + throw new DataValidationException(prepareMsg("Can't assign %s with empty data!", type)); + } + if (!otaPackage.getDeviceProfileId().equals(deviceProfileId)) { + throw new DataValidationException(prepareMsg("Can't assign %s with different deviceProfile!", type)); + } + } + + private String prepareMsg(String msg, OtaPackageType type) { + return String.format(msg, type.name().toLowerCase()); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java index 23e520f070..d30180a892 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceDataValidator.java @@ -22,17 +22,13 @@ import org.springframework.util.StringUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.device.data.DeviceTransportConfiguration; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.device.DeviceDao; import org.thingsboard.server.dao.exception.DataValidationException; -import org.thingsboard.server.dao.ota.OtaPackageService; -import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; @@ -41,7 +37,7 @@ import java.util.Optional; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @Component -public class DeviceDataValidator extends DataValidator { +public class DeviceDataValidator extends AbstractHasOtaPackageValidator { @Autowired private DeviceDao deviceDao; @@ -56,9 +52,6 @@ public class DeviceDataValidator extends DataValidator { @Lazy private TbTenantProfileCache tenantProfileCache; - @Autowired - private OtaPackageService otaPackageService; - @Override protected void validateCreate(TenantId tenantId, Device device) { DefaultTenantProfileConfiguration profileConfiguration = @@ -103,36 +96,6 @@ public class DeviceDataValidator extends DataValidator { .flatMap(deviceData -> Optional.ofNullable(deviceData.getTransportConfiguration())) .ifPresent(DeviceTransportConfiguration::validate); - if (device.getFirmwareId() != null) { - OtaPackage firmware = otaPackageService.findOtaPackageById(tenantId, device.getFirmwareId()); - if (firmware == null) { - throw new DataValidationException("Can't assign non-existent firmware!"); - } - if (!firmware.getType().equals(OtaPackageType.FIRMWARE)) { - throw new DataValidationException("Can't assign firmware with type: " + firmware.getType()); - } - if (firmware.getData() == null && !firmware.hasUrl()) { - throw new DataValidationException("Can't assign firmware with empty data!"); - } - if (!firmware.getDeviceProfileId().equals(device.getDeviceProfileId())) { - throw new DataValidationException("Can't assign firmware with different deviceProfile!"); - } - } - - if (device.getSoftwareId() != null) { - OtaPackage software = otaPackageService.findOtaPackageById(tenantId, device.getSoftwareId()); - if (software == null) { - throw new DataValidationException("Can't assign non-existent software!"); - } - if (!software.getType().equals(OtaPackageType.SOFTWARE)) { - throw new DataValidationException("Can't assign software with type: " + software.getType()); - } - if (software.getData() == null && !software.hasUrl()) { - throw new DataValidationException("Can't assign software with empty data!"); - } - if (!software.getDeviceProfileId().equals(device.getDeviceProfileId())) { - throw new DataValidationException("Can't assign firmware with different deviceProfile!"); - } - } + validateOtaPackage(tenantId, device, device.getDeviceProfileId()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index b1fb9b150b..eec7bc8bc8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -35,8 +35,8 @@ import org.springframework.util.CollectionUtils; import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileProvisionType; -import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.device.credentials.lwm2m.LwM2MSecurityMode; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; @@ -52,7 +52,6 @@ import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.LwM2MBo import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.RPKLwM2MBootstrapServerCredential; import org.thingsboard.server.common.data.device.profile.lwm2m.bootstrap.X509LwM2MBootstrapServerCredential; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.queue.Queue; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.msg.EncryptionUtil; @@ -62,10 +61,9 @@ import org.thingsboard.server.dao.device.DeviceProfileDao; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; -import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.queue.QueueService; import org.thingsboard.server.dao.rule.RuleChainService; -import org.thingsboard.server.dao.service.DataValidator; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; import java.util.HashSet; @@ -74,7 +72,7 @@ import java.util.Set; import java.util.stream.Collectors; @Component -public class DeviceProfileDataValidator extends DataValidator { +public class DeviceProfileDataValidator extends AbstractHasOtaPackageValidator { private static final Location LOCATION = new Location("", "", -1, -1); private static final String ATTRIBUTES_PROTO_SCHEMA = "attributes proto schema"; @@ -94,11 +92,11 @@ public class DeviceProfileDataValidator extends DataValidator { @Autowired private QueueService queueService; @Autowired - private OtaPackageService otaPackageService; - @Autowired private RuleChainService ruleChainService; @Autowired private DashboardService dashboardService; + @Autowired + private TbTenantProfileCache tenantProfileCache; private static String invalidSchemaProvidedMessage(String schemaName) { return "[Transport Configuration] invalid " + schemaName + " provided!"; @@ -133,6 +131,11 @@ public class DeviceProfileDataValidator extends DataValidator { if (queue == null) { throw new DataValidationException("Device profile is referencing to non-existent queue!"); } + TenantProfile tenantProfile = tenantProfileCache.get(deviceProfile.getTenantId()); + if ((tenantProfile.isIsolatedTbRuleEngine() && !queue.getTenantId().equals(deviceProfile.getTenantId())) + || (!tenantProfile.isIsolatedTbRuleEngine() && !queue.getTenantId().isNullUid())) { + throw new DataValidationException("Can't assign queue from different tenant!"); + } } if (deviceProfile.getProvisionType() == null) { deviceProfile.setProvisionType(DeviceProfileProvisionType.DISABLED); @@ -192,6 +195,9 @@ public class DeviceProfileDataValidator extends DataValidator { if (ruleChain == null) { throw new DataValidationException("Can't assign non-existent rule chain!"); } + if (!ruleChain.getTenantId().equals(deviceProfile.getTenantId())) { + throw new DataValidationException("Can't assign rule chain from different tenant!"); + } } if (deviceProfile.getDefaultDashboardId() != null) { @@ -199,39 +205,12 @@ public class DeviceProfileDataValidator extends DataValidator { if (dashboard == null) { throw new DataValidationException("Can't assign non-existent dashboard!"); } - } - - if (deviceProfile.getFirmwareId() != null) { - OtaPackage firmware = otaPackageService.findOtaPackageById(tenantId, deviceProfile.getFirmwareId()); - if (firmware == null) { - throw new DataValidationException("Can't assign non-existent firmware!"); - } - if (!firmware.getType().equals(OtaPackageType.FIRMWARE)) { - throw new DataValidationException("Can't assign firmware with type: " + firmware.getType()); - } - if (firmware.getData() == null && !firmware.hasUrl()) { - throw new DataValidationException("Can't assign firmware with empty data!"); - } - if (!firmware.getDeviceProfileId().equals(deviceProfile.getId())) { - throw new DataValidationException("Can't assign firmware with different deviceProfile!"); + if (!dashboard.getTenantId().equals(deviceProfile.getTenantId())) { + throw new DataValidationException("Can't assign dashboard from different tenant!"); } } - if (deviceProfile.getSoftwareId() != null) { - OtaPackage software = otaPackageService.findOtaPackageById(tenantId, deviceProfile.getSoftwareId()); - if (software == null) { - throw new DataValidationException("Can't assign non-existent software!"); - } - if (!software.getType().equals(OtaPackageType.SOFTWARE)) { - throw new DataValidationException("Can't assign software with type: " + software.getType()); - } - if (software.getData() == null && !software.hasUrl()) { - throw new DataValidationException("Can't assign software with empty data!"); - } - if (!software.getDeviceProfileId().equals(deviceProfile.getId())) { - throw new DataValidationException("Can't assign firmware with different deviceProfile!"); - } - } + validateOtaPackage(tenantId, deviceProfile, deviceProfile.getId()); } @Override From 4c919cfe015793b230592da305eaef9ec10faa80 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 7 Jul 2022 22:28:17 +0200 Subject: [PATCH 21/28] fixed circular dependency --- .../org/thingsboard/server/dao/tenant/TenantServiceImpl.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 6da185ed19..058a24c856 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -99,6 +99,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService Date: Fri, 8 Jul 2022 10:11:13 +0300 Subject: [PATCH 22/28] blackbox tests: wait for js-executor kafka message --- .../java/org/thingsboard/server/msa/ContainerTestSuite.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 1fbf3755e6..6046b4db52 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -48,6 +48,7 @@ public class ContainerTestSuite { private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; + private static final String TB_JS_EXECUTOR_KAFKA_LOG_REGEXP = ".*Consumer has joined the group.*"; private static DockerComposeContainer testContainer; @@ -111,7 +112,8 @@ public class ContainerTestSuite { .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_KAFKA_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); } catch (Exception e) { log.error("Failed to create test container", e); fail("Failed to create test container"); From bbfca14302a85ff8f3a1fbb635a8db53f30b65cc Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Fri, 8 Jul 2022 10:56:46 +0300 Subject: [PATCH 23/28] Fix incorrect shutdown JavaScript Executor RabbitMQ queue --- msa/js-executor/queue/rabbitmqTemplate.ts | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/msa/js-executor/queue/rabbitmqTemplate.ts b/msa/js-executor/queue/rabbitmqTemplate.ts index 9369f170f5..ccd3cef54b 100644 --- a/msa/js-executor/queue/rabbitmqTemplate.ts +++ b/msa/js-executor/queue/rabbitmqTemplate.ts @@ -20,7 +20,6 @@ import { JsInvokeMessageProcessor } from '../api/jsInvokeMessageProcessor' import { IQueue } from './queue.models'; import amqp, { ConfirmChannel, Connection } from 'amqplib'; import { Options, Replies } from 'amqplib/properties'; -import { sleep } from '../api/utils'; export class RabbitMqTemplate implements IQueue { @@ -32,7 +31,6 @@ export class RabbitMqTemplate implements IQueue { private username = config.get('rabbitmq.username'); private password = config.get('rabbitmq.password'); private queueProperties: string = config.get('rabbitmq.queue_properties'); - private pollInterval = Number(config.get('js.response_poll_interval')); private queueOptions: Options.AssertQueue = { durable: false, @@ -41,7 +39,6 @@ export class RabbitMqTemplate implements IQueue { }; private connection: Connection; private channel: ConfirmChannel; - private stopped = false; private topics: string[] = []; name = 'RabbitMQ'; @@ -60,20 +57,12 @@ export class RabbitMqTemplate implements IQueue { const messageProcessor = new JsInvokeMessageProcessor(this); - while (!this.stopped) { - let pollStartTs = new Date().getTime(); - let message = await this.channel.get(this.requestTopic); - + await this.channel.consume(this.requestTopic, (message) => { if (message) { messageProcessor.onJsInvokeMessage(JSON.parse(message.content.toString('utf8'))); this.channel.ack(message); - } else { - let pollDuration = new Date().getTime() - pollStartTs; - if (pollDuration < this.pollInterval) { - await sleep(this.pollInterval - pollDuration); - } } - } + }) } async send(responseTopic: string, scriptId: string, rawResponse: Buffer, headers: any): Promise { From acf5e130e9c4fcec1a266bed9d84e29ec78661f5 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 8 Jul 2022 12:07:36 +0300 Subject: [PATCH 24/28] Support of aws sqs and rabbitmq for bbt --- docker/docker-compose.rabbitmq-server.yml | 69 ++++++++++++++++ .../server/msa/ContainerTestSuite.java | 82 ++++++++++++++----- 2 files changed, 131 insertions(+), 20 deletions(-) create mode 100644 docker/docker-compose.rabbitmq-server.yml diff --git a/docker/docker-compose.rabbitmq-server.yml b/docker/docker-compose.rabbitmq-server.yml new file mode 100644 index 0000000000..21aba7061a --- /dev/null +++ b/docker/docker-compose.rabbitmq-server.yml @@ -0,0 +1,69 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + rabbitmq: + restart: always + image: rabbitmq:3 + ports: + - '5672:5672' + environment: + RABBITMQ_DEFAULT_USER: YOUR_USERNAME + RABBITMQ_DEFAULT_PASS: YOUR_PASSWORD + tb-js-executor: + depends_on: + - rabbitmq + tb-core1: + depends_on: + - rabbitmq + tb-core2: + depends_on: + - rabbitmq + tb-rule-engine1: + depends_on: + - rabbitmq + tb-rule-engine2: + depends_on: + - rabbitmq + tb-mqtt-transport1: + depends_on: + - rabbitmq + tb-mqtt-transport2: + depends_on: + - rabbitmq + tb-http-transport1: + depends_on: + - rabbitmq + tb-http-transport2: + depends_on: + - rabbitmq + tb-coap-transport: + depends_on: + - rabbitmq + tb-lwm2m-transport: + depends_on: + - rabbitmq + tb-snmp-transport: + depends_on: + - rabbitmq + tb-vc-executor1: + depends_on: + - rabbitmq + tb-vc-executor2: + depends_on: + - rabbitmq diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 1fbf3755e6..0c25aa15f0 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -22,14 +22,20 @@ import org.junit.extensions.cpsuite.ClasspathSuite; import org.junit.runner.RunWith; import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.thingsboard.server.common.data.StringUtils; import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.UUID; import static org.hamcrest.CoreMatchers.containsString; @@ -44,10 +50,12 @@ import static org.junit.Assert.fail; public class ContainerTestSuite { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); + final static String QUEUE_TYPE = System.getProperty("blackBoxTests.queue", "kafka"); private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; + private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(120); private static DockerComposeContainer testContainer; @@ -81,17 +89,33 @@ public class ContainerTestSuite { List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), - IS_HYBRID_MODE - ? new File(targetDir + "docker-compose.hybrid.yml") - : new File(targetDir + "docker-compose.postgres.yml"), + new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose.kafka.yml"), - IS_REDIS_CLUSTER - ? new File(targetDir + "docker-compose.redis-cluster.yml") - : new File(targetDir + "docker-compose.redis.yml"), - IS_REDIS_CLUSTER - ? new File(targetDir + "docker-compose.redis-cluster.volumes.yml") - : new File(targetDir + "docker-compose.redis.volumes.yml"))); + new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), + new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), + new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")) + )); + + Map queueEnv = new HashMap<>(); + queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE); + switch (QUEUE_TYPE) { + case "kafka": + composeFiles.add(new File(targetDir + "docker-compose.kafka.yml")); + break; + case "aws-sqs": + replaceInFile(targetDir, "queue-aws-sqs.env", + Map.of("YOUR_KEY", getSysProp("blackBoxTests.awsKey"), + "YOUR_SECRET", "blackBoxTests.awsSecret", + "YOUR_REGION", "blackBoxTests.awsRegion")); + break; + case "rabbitmq": + composeFiles.add(new File(targetDir + "docker-compose.rabbitmq-server.yml")); + replaceInFile(targetDir, "queue-rabbitmq.env", + Map.of("localhost", "rabbitmq")); + break; + default: + throw new RuntimeException("Unsupported queue type: " + QUEUE_TYPE); + } if (IS_HYBRID_MODE) { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); @@ -102,16 +126,17 @@ public class ContainerTestSuite { .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) .withEnv(installTb.getEnv()) + .withEnv(queueEnv) .withEnv("LOAD_BALANCER_NAME", "") - .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); + .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); } catch (Exception e) { log.error("Failed to create test container", e); fail("Failed to create test container"); @@ -120,6 +145,23 @@ public class ContainerTestSuite { return testContainer; } + private static void replaceInFile(String targetDir, String fileName, Map replacements) throws IOException { + Path envFilePath = Path.of(targetDir, fileName); + String data = Files.readString(envFilePath); + for (var entry : replacements.entrySet()) { + data = data.replace(entry.getKey(), entry.getValue()); + } + Files.write(envFilePath, data.getBytes(StandardCharsets.UTF_8)); + } + + private static String getSysProp(String propertyName) { + var value = System.getProperty(propertyName); + if (StringUtils.isEmpty(value)) { + throw new RuntimeException("Please define system property: " + propertyName + "!"); + } + return value; + } + private static void tryDeleteDir(String targetDir) { try { log.info("Trying to delete temp dir {}", targetDir); @@ -135,7 +177,7 @@ public class ContainerTestSuite { * docker-compose files which contain container_name are not supported and the creation of DockerComposeContainer fails due to IllegalStateException. * This has been introduced in #1151 as a quick fix for unintuitive feedback. https://github.com/testcontainers/testcontainers-java/issues/1151 * Using the latest testcontainers and waiting for the fix... - * */ + */ private static void replaceInFile(String sourceFilename, String target, String replacement, String verifyPhrase) { try { File file = new File(sourceFilename); From 01528d96bdc989e151bc5c10b23d5aea1f51f66d Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 8 Jul 2022 12:10:02 +0300 Subject: [PATCH 25/28] Fix timeout parameter --- .../java/org/thingsboard/server/msa/ContainerTestSuite.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 0c25aa15f0..e3ff4d9af3 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -55,7 +55,7 @@ public class ContainerTestSuite { private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; - private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(120); + private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); private static DockerComposeContainer testContainer; From 54e94ce0ba8b172c9f7988f6c474c0522e26ae22 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 8 Jul 2022 12:48:08 +0300 Subject: [PATCH 26/28] Fix HashPartitionServiceTest --- .../server/queue/discovery/HashPartitionServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java index e20f40be46..88aa45c74c 100644 --- a/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/queue/discovery/HashPartitionServiceTest.java @@ -112,7 +112,7 @@ public class HashPartitionServiceTest { map.put(partition, map.getOrDefault(partition, 0) + 1); } - checkDispersion(start, map, ITERATIONS, 1.0); + checkDispersion(start, map, ITERATIONS, 5.0); } @SneakyThrows From 854550b7d556d329cdbbea95d59d63da25abfe03 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 8 Jul 2022 12:48:47 +0300 Subject: [PATCH 27/28] Improved log expression to check for JS executors --- .../org/thingsboard/server/msa/ContainerTestSuite.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index a1c8468c4f..3706b7a303 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -32,7 +32,6 @@ import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -55,7 +54,7 @@ public class ContainerTestSuite { private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; - private static final String TB_JS_EXECUTOR_KAFKA_LOG_REGEXP = ".*Consumer has joined the group.*"; + private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); private static DockerComposeContainer testContainer; @@ -137,8 +136,8 @@ public class ContainerTestSuite { .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_KAFKA_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); } catch (Exception e) { log.error("Failed to create test container", e); fail("Failed to create test container"); From 293ac8d16436e7422ea97f6f706bd81b863423c8 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 8 Jul 2022 12:59:13 +0300 Subject: [PATCH 28/28] Moved rabbit-mq server to the test resources --- .../java/org/thingsboard/server/msa/ContainerTestSuite.java | 2 ++ .../src/test/resources}/docker-compose.rabbitmq-server.yml | 0 2 files changed, 2 insertions(+) rename {docker => msa/black-box-tests/src/test/resources}/docker-compose.rabbitmq-server.yml (100%) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 3706b7a303..60934bdbbf 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -74,6 +74,8 @@ public class ContainerTestSuite { FileUtils.copyDirectory(new File(SOURCE_DIR), new File(targetDir)); replaceInFile(targetDir + "docker-compose.yml", " container_name: \"${LOAD_BALANCER_NAME}\"", "", "container_name"); + FileUtils.copyDirectory(new File("src/test/resources"), new File(targetDir)); + class DockerComposeContainerImpl> extends DockerComposeContainer { public DockerComposeContainerImpl(List composeFiles) { super(composeFiles); diff --git a/docker/docker-compose.rabbitmq-server.yml b/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml similarity index 100% rename from docker/docker-compose.rabbitmq-server.yml rename to msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml