Merge pull request #8143 from smatvienko-tb/feature/test-improvements-memory

[3.5] Test improvements - memory

application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java

@@ -29,8 +29,10 @@ import lombok.extern.slf4j.Slf4j;
 import org.hamcrest.Matcher;
 import org.hibernate.exception.ConstraintViolationException;
 import org.junit.After;
+import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.rules.TestRule;
 import org.junit.rules.TestWatcher;
@@ -209,9 +211,20 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
                 this.mappingJackson2HttpMessageConverter);
     }
 
+    @BeforeClass
+    public static void beforeWebTestClass() throws Exception {
+
+    }
+
+    @AfterClass
+    public static void afterWebTestClass() throws Exception {
+        Mockito.clearAllCaches();
+    }
+
+
     @Before
     public void setupWebTest() throws Exception {
-        log.info("Executing web test setup");
+        log.debug("Executing web test setup");
 
         setupMailServiceMock();
 
@@ -253,7 +266,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
 
         resetTokens();
 
-        log.info("Executed web test setup");
+        log.debug("Executed web test setup");
     }
 
     private void setupMailServiceMock() throws ThingsboardException {
@@ -279,7 +292,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest {
 
     @After
     public void teardownWebTest() throws Exception {
-        log.info("Executing web test teardown");
+        log.debug("Executing web test teardown");
 
         loginSysAdmin();
         doDelete("/api/tenant/" + tenantId.getId().toString())

application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java

@@ -192,7 +192,7 @@ public abstract class BaseWidgetsBundleControllerTest extends AbstractController
         WidgetsBundle savedWidgetsBundle = doPost("/api/widgetsBundle", widgetsBundle, WidgetsBundle.class);
         savedWidgetsBundle.setAlias("new_alias");
 
-        Mockito.reset(tbClusterService);
+        Mockito.clearInvocations(tbClusterService);
 
         doPost("/api/widgetsBundle", savedWidgetsBundle)
                 .andExpect(status().isBadRequest())

application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java

@@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 @DaoSqlTest
 @TestPropertySource(properties = {
+        "js.evaluator=local",
         "js.max_script_body_size=50",
         "js.max_total_args_size=50",
         "js.max_result_size=50",

application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java

@@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 @Slf4j
 public abstract class AbstractTransportIntegrationTest extends AbstractControllerTest {
 
-    protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 10;
+    protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 30;
 
     protected static final String MQTT_URL = "tcp://localhost:1883";
     protected static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/";

application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java

@@ -45,6 +45,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
 @TestPropertySource(properties = {
+        "coap.enabled=true",
+        "service.integrations.supported=ALL",
         "transport.coap.enabled=true",
 })
 @Slf4j

application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java

@@ -46,8 +46,8 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 
 @TestPropertySource(properties = {
+        "service.integrations.supported=ALL",
         "transport.mqtt.enabled=true",
-        "js.evaluator=mock",
 })
 @Slf4j
 public abstract class AbstractMqttIntegrationTest extends AbstractTransportIntegrationTest {

application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java

@@ -128,14 +128,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
         client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE);
 
         doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
-        onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await onUpdateCallback").isTrue();
 
         validateUpdateAttributesJsonResponse(onUpdateCallback, SHARED_ATTRIBUTES_PAYLOAD);
 
         MqttTestCallback onDeleteCallback = new MqttTestCallback();
         client.setCallback(onDeleteCallback);
         doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class);
-        onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await onDeleteCallback").isTrue();
         validateUpdateAttributesJsonResponse(onDeleteCallback, SHARED_ATTRIBUTES_DELETED_RESPONSE);
         client.disconnect();
     }
@@ -148,13 +150,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
         client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE);
 
         doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
-        onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await onUpdateCallback").isTrue();
         validateUpdateAttributesProtoResponse(onUpdateCallback);
 
         MqttTestCallback onDeleteCallback = new MqttTestCallback();
         client.setCallback(onDeleteCallback);
         doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class);
-        onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await onDeleteCallback").isTrue();
         validateDeleteAttributesProtoResponse(onDeleteCallback);
         client.disconnect();
     }
@@ -165,7 +169,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateUpdateAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException {
-        assertNotNull(callback.getPayloadBytes());
+        assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull();
         TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
         List<TransportProtos.TsKvProto> tsKvProtoList = getTsKvProtoList("shared");
         attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList);
@@ -181,7 +185,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateDeleteAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException {
-        assertNotNull(callback.getPayloadBytes());
+        assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull();
         TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
         attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson");
 
@@ -212,7 +216,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
         client.subscribeAndWait(GATEWAY_ATTRIBUTES_TOPIC, MqttQoS.AT_MOST_ONCE);
 
         doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
-        onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await onUpdateCallback").isTrue();
 
         validateJsonGatewayUpdateAttributesResponse(onUpdateCallback, deviceName, SHARED_ATTRIBUTES_PAYLOAD);
 
@@ -220,7 +225,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
         client.setCallback(onDeleteCallback);
 
         doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class);
-        onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await onDeleteCallback").isTrue();
 
         validateJsonGatewayUpdateAttributesResponse(onDeleteCallback, deviceName, SHARED_ATTRIBUTES_DELETED_RESPONSE);
         client.disconnect();
@@ -249,7 +255,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateJsonGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName, String expectResultData) {
-        assertNotNull(callback.getPayloadBytes());
+        assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull();
         assertEquals(JacksonUtil.toJsonNode(getGatewayAttributesResponseJson(deviceName, expectResultData)), JacksonUtil.fromBytes(callback.getPayloadBytes()));
     }
 
@@ -263,8 +269,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateProtoGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException {
-        callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
-        assertNotNull(callback.getPayloadBytes());
+                .as("await callback").isTrue();
+        assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull();
 
         TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
         List<TransportProtos.TsKvProto> tsKvProtoList = getTsKvProtoList("shared");
@@ -288,8 +295,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateProtoGatewayDeleteAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException {
-        callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
-        assertNotNull(callback.getPayloadBytes());
+                .as("await callback").isTrue();
+        assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull();
         TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
         attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson");
         TransportProtos.AttributeUpdateNotificationMsg attributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build();
@@ -551,13 +559,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateJsonResponse(MqttTestCallback callback, String expectedResponse) throws InterruptedException {
-        callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await callback").isTrue();
         assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS());
         assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(callback.getPayloadBytes()));
     }
 
     protected void validateProtoResponse(MqttTestCallback callback, TransportProtos.GetAttributeResponseMsg expectedResponse) throws InterruptedException, InvalidProtocolBufferException {
-        callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await callback").isTrue();
         assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS());
         TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(callback.getPayloadBytes());
         assertEquals(expectedResponse.getRequestId(), actualAttributesResponse.getRequestId());
@@ -580,14 +590,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateJsonResponseGateway(MqttTestCallback callback, String deviceName, String expectedValues) throws InterruptedException {
-        callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await callback").isTrue();
         assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS());
         String expectedRequestPayload = "{\"id\":1,\"device\":\"" + deviceName + "\",\"values\":" + expectedValues + "}";
         assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.fromBytes(callback.getPayloadBytes()));
     }
 
     protected void validateProtoClientResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException {
-        callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await callback").isTrue();
         assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS());
         TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, true);
         TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes());
@@ -603,7 +615,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt
     }
 
     protected void validateProtoSharedResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException {
-        callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
+        assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS))
+                .as("await callback").isTrue();
         assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS());
         TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, false);
         TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes());

application/src/test/resources/application-test.properties

@@ -1,3 +1,4 @@
+js.evaluator=mock
 transport.lwm2m.server.security.credentials.enabled=true
 transport.lwm2m.server.security.credentials.type=KEYSTORE
 transport.lwm2m.server.security.credentials.keystore.store_file=lwm2m/credentials/lwm2mserver.jks
@@ -20,6 +21,9 @@ transport.mqtt.enabled=false
 transport.coap.enabled=false
 transport.lwm2m.enabled=false
 transport.snmp.enabled=false
+coap.enabled=false
+integrations.rpc.enabled=false
+service.integrations.supported=NONE
 
 # Low latency settings to perform tests as fast as possible
 sql.attributes.batch_max_delay=5
@@ -63,3 +67,4 @@ sql.ttl.audit_logs.ttl=2592000
 
 sql.edge_events.partition_size=168
 sql.ttl.edge_events.edge_event_ttl=2592000
+spring.test.context.cache.maxSize=1

application/src/test/resources/logback-test.xml

@@ -15,6 +15,7 @@
     <logger name="org.apache.cassandra" level="WARN"/>
     <logger name="org.cassandraunit" level="INFO"/>
     <logger name="org.eclipse.leshan" level="INFO"/>
+    <logger name="org.thingsboard.server.controller.AbstractWebTest" level="INFO"/>
 
     <!-- mute TelemetryEdgeSqlTest that causes a lot of randomly generated errors -->
     <logger name="org.thingsboard.server.service.edge.rpc.EdgeGrpcSession" level="OFF"/>

common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java

@@ -129,6 +129,9 @@ public class NashornJsInvokeService extends AbstractJsInvokeService {
         if (monitorExecutorService != null) {
             monitorExecutorService.shutdownNow();
         }
+        if (jsExecutor != null) {
+            jsExecutor.shutdownNow();
+        }
     }
 
     @Override

dao/src/test/resources/nosql-test.properties

@@ -15,7 +15,7 @@ spring.datasource.username=postgres
 spring.datasource.password=postgres
 spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb
 spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver
-spring.datasource.hikari.maximumPoolSize = 50
+spring.datasource.hikari.maximumPoolSize=16
 
 queue.rule-engine.queues[0].name=Main
 queue.rule-engine.queues[0].topic=tb_rule_engine.main

dao/src/test/resources/sql-test.properties

@@ -16,7 +16,7 @@ spring.datasource.username=postgres
 spring.datasource.password=postgres
 spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb
 spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver
-spring.datasource.hikari.maximumPoolSize = 50
+spring.datasource.hikari.maximumPoolSize=16
 
 service.type=monolith
 

msa/pom.xml

@@ -39,13 +39,14 @@
     </properties>
 
     <modules>
+        <!--Modules order is important to speedup parallel build and avoid yarn pgk parallel execution-->
         <module>tb</module>
+        <module>web-ui</module>
         <module>vc-executor</module>
         <module>vc-executor-docker</module>
-        <module>js-executor</module>
-        <module>web-ui</module>
         <module>tb-node</module>
         <module>transport</module>
+        <module>js-executor</module>
     </modules>
 
     <profiles>

pom.xml

@@ -85,7 +85,7 @@
         <netty-tcnative.version>2.0.51.Final</netty-tcnative.version>
         <os-maven-plugin.version>1.7.0</os-maven-plugin.version>
         <rabbitmq.version>4.8.0</rabbitmq.version>
-        <surefire.version>3.0.0-M6</surefire.version>
+        <surefire.version>3.0.0-M9</surefire.version>
         <jar-plugin.version>3.0.2</jar-plugin.version>
         <springfox-swagger.version>3.0.4</springfox-swagger.version>
         <swagger-annotations.version>1.6.3</swagger-annotations.version>
@@ -659,7 +659,7 @@
                     <version>${surefire.version}</version>
                     <configuration>
                         <argLine>
-                            --illegal-access=permit
+                            --illegal-access=permit -Xss384k -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=20
                         </argLine>
                     </configuration>
                 </plugin>