From 34a587ff02be4f975eb0d17ed7f362591e583487 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 15 Jul 2021 10:39:58 +0300 Subject: [PATCH 1/9] Update frontend-maven-plugin --- msa/js-executor/pom.xml | 2 -- msa/web-ui/pom.xml | 2 -- pom.xml | 5 +++++ ui-ngx/pom.xml | 3 --- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index a254b1acde..03ccd23714 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -59,7 +59,6 @@ com.github.eirslett frontend-maven-plugin - 1.7.5 target ${basedir} @@ -149,7 +148,6 @@ com.github.eirslett frontend-maven-plugin - 1.7.5 target ${basedir} diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index 85f5db6ce3..d9b99a5a50 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -68,7 +68,6 @@ com.github.eirslett frontend-maven-plugin - 1.7.5 target ${basedir} @@ -196,7 +195,6 @@ com.github.eirslett frontend-maven-plugin - 1.7.5 target ${basedir} diff --git a/pom.xml b/pom.xml index 248baec7cd..8cd9629c83 100755 --- a/pom.xml +++ b/pom.xml @@ -635,6 +635,11 @@ gradle-maven-plugin 1.0.10 + + com.github.eirslett + frontend-maven-plugin + 1.12.0 + org.apache.maven.plugins maven-surefire-plugin diff --git a/ui-ngx/pom.xml b/ui-ngx/pom.xml index fb8c809260..73249759bc 100644 --- a/ui-ngx/pom.xml +++ b/ui-ngx/pom.xml @@ -45,7 +45,6 @@ com.github.eirslett frontend-maven-plugin - 1.7.5 target ${basedir} @@ -85,7 +84,6 @@ com.github.eirslett frontend-maven-plugin - 1.7.5 target ${basedir} @@ -117,7 +115,6 @@ com.github.eirslett frontend-maven-plugin - 1.7.5 target ${basedir} From dae0eb52124475fe6c37b82479d2761a1072e6f2 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 15 Jul 2021 11:20:49 +0300 Subject: [PATCH 2/9] Add transactional support for HSQLDB --- dao/src/main/resources/sql/schema-entities-hsql.sql | 3 +++ dao/src/main/resources/sql/schema-ts-hsql.sql | 1 + 2 files changed, 4 insertions(+) diff --git a/dao/src/main/resources/sql/schema-entities-hsql.sql b/dao/src/main/resources/sql/schema-entities-hsql.sql index 812f65fce5..be227828d6 100644 --- a/dao/src/main/resources/sql/schema-entities-hsql.sql +++ b/dao/src/main/resources/sql/schema-entities-hsql.sql @@ -14,6 +14,9 @@ -- limitations under the License. -- +SET DATABASE SQL SYNTAX PGS TRUE; +SET DATABASE TRANSACTION CONTROL MVCC; + CREATE TABLE IF NOT EXISTS admin_settings ( id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY, created_time bigint NOT NULL, diff --git a/dao/src/main/resources/sql/schema-ts-hsql.sql b/dao/src/main/resources/sql/schema-ts-hsql.sql index 2897938c67..3a54438340 100644 --- a/dao/src/main/resources/sql/schema-ts-hsql.sql +++ b/dao/src/main/resources/sql/schema-ts-hsql.sql @@ -15,6 +15,7 @@ -- SET DATABASE SQL SYNTAX PGS TRUE; +SET DATABASE TRANSACTION CONTROL MVCC; CREATE TABLE IF NOT EXISTS ts_kv ( entity_id uuid NOT NULL, From c149f1ac38a2eca52b924b3731babc7bf7cddc92 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 15 Jul 2021 11:55:49 +0300 Subject: [PATCH 3/9] Black box tests dependencies --- msa/black-box-tests/pom.xml | 16 ++++++++++++++++ pom.xml | 24 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 5c6740e61d..2fb813916f 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -92,6 +92,22 @@ org.thingsboard rest-client + + org.thingsboard.msa + js-executor + + + org.thingsboard.msa + web-ui + + + org.thingsboard.msa + tb-node + + + org.thingsboard.msa + transport + diff --git a/pom.xml b/pom.xml index 8cd9629c83..a2adfb62d4 100755 --- a/pom.xml +++ b/pom.xml @@ -965,6 +965,30 @@ ${project.version} test + + org.thingsboard.msa + js-executor + ${project.version} + test + + + org.thingsboard.msa + web-ui + ${project.version} + test + + + org.thingsboard.msa + tb-node + ${project.version} + test + + + org.thingsboard.msa + transport + ${project.version} + test + org.thingsboard dao From 559aa282ca0f12f949d6d1983b6744bf77b854c4 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 15 Jul 2021 13:18:57 +0300 Subject: [PATCH 4/9] Fix black box dependencies --- msa/black-box-tests/pom.xml | 29 ++++++++++++++++++++++++++--- msa/pom.xml | 7 +++++++ pom.xml | 36 ++++++++++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 2fb813916f..e94b066fb8 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -95,21 +95,44 @@ org.thingsboard.msa js-executor + docker-info org.thingsboard.msa web-ui + docker-info org.thingsboard.msa tb-node + docker-info - org.thingsboard.msa - transport + org.thingsboard.msa.transport + coap + docker-info + + + org.thingsboard.msa.transport + http + docker-info + + + org.thingsboard.msa.transport + mqtt + docker-info + + + org.thingsboard.msa.transport + lwm2m + docker-info + + + org.thingsboard.msa.transport + snmp + docker-info - diff --git a/msa/pom.xml b/msa/pom.xml index 3d7bf3f347..f0546fb71c 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -45,6 +45,13 @@ + + + com.spotify + dockerfile-maven-extension + 1.4.13 + + diff --git a/pom.xml b/pom.xml index a2adfb62d4..6d54fcdaa5 100755 --- a/pom.xml +++ b/pom.xml @@ -969,24 +969,56 @@ org.thingsboard.msa js-executor ${project.version} + docker-info test org.thingsboard.msa web-ui ${project.version} + docker-info test org.thingsboard.msa tb-node ${project.version} + docker-info test - org.thingsboard.msa - transport + org.thingsboard.msa.transport + coap + ${project.version} + docker-info + test + + + org.thingsboard.msa.transport + http + ${project.version} + docker-info + test + + + org.thingsboard.msa.transport + mqtt + ${project.version} + docker-info + test + + + org.thingsboard.msa.transport + lwm2m + ${project.version} + docker-info + test + + + org.thingsboard.msa.transport + snmp ${project.version} + docker-info test From 050f0d9b072ea2732f9303ab912ac9efd588fa46 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 15 Jul 2021 20:35:13 +0300 Subject: [PATCH 5/9] fixed duplicated read/observe requests --- .../lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 257fff72eb..26e457de17 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -422,6 +422,7 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl private void sendReadRequests(LwM2mClient lwM2MClient, Lwm2mDeviceProfileTransportConfiguration profile, Set supportedObjects) { Set targetIds = new HashSet<>(profile.getObserveAttr().getAttribute()); targetIds.addAll(profile.getObserveAttr().getTelemetry()); + targetIds = diffSets(targetIds, profile.getObserveAttr().getObserve()); targetIds = targetIds.stream().filter(target -> isSupportedTargetId(supportedObjects, target)).collect(Collectors.toSet()); CountDownLatch latch = new CountDownLatch(targetIds.size()); From be86a2df480e80a9957cdb6e88af540bcfce6c85 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Fri, 16 Jul 2021 11:41:19 +0300 Subject: [PATCH 6/9] fixed sendReadRequests --- .../lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java index 26e457de17..799ba54b4a 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/uplink/DefaultLwM2MUplinkMsgHandler.java @@ -422,7 +422,7 @@ public class DefaultLwM2MUplinkMsgHandler extends LwM2MExecutorAwareService impl private void sendReadRequests(LwM2mClient lwM2MClient, Lwm2mDeviceProfileTransportConfiguration profile, Set supportedObjects) { Set targetIds = new HashSet<>(profile.getObserveAttr().getAttribute()); targetIds.addAll(profile.getObserveAttr().getTelemetry()); - targetIds = diffSets(targetIds, profile.getObserveAttr().getObserve()); + targetIds = diffSets(profile.getObserveAttr().getObserve(), targetIds); targetIds = targetIds.stream().filter(target -> isSupportedTargetId(supportedObjects, target)).collect(Collectors.toSet()); CountDownLatch latch = new CountDownLatch(targetIds.size()); From 085044e6e79cfa5b1487ee4040dcc944b847e8f9 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 16 Jul 2021 12:21:00 +0300 Subject: [PATCH 7/9] Exclude black box tests module from build when black box tests are not enabled. --- msa/black-box-tests/pom.xml | 1 - msa/pom.xml | 22 +++++++++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index e94b066fb8..7840b2ecf2 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -34,7 +34,6 @@ UTF-8 ${basedir}/../.. - true 1.11.4 1.10 4.5.13 diff --git a/msa/pom.xml b/msa/pom.xml index f0546fb71c..79dce4ae87 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -33,6 +33,8 @@ ${basedir}/.. thingsboard true + true + 1.4.13 @@ -41,15 +43,29 @@ web-ui tb-node transport - black-box-tests + + + black-box-tests + + + blackBoxTests.skip + false + + + + black-box-tests + + + + com.spotify dockerfile-maven-extension - 1.4.13 + ${dockerfile-maven.version} @@ -57,7 +73,7 @@ com.spotify dockerfile-maven-plugin - 1.4.13 + ${dockerfile-maven.version} From bd3e2618149a8ec6311cfac45ee2360a6e918f30 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 16 Jul 2021 19:16:42 +0300 Subject: [PATCH 8/9] MQTT attributes tests improvements --- ...tMqttAttributesRequestIntegrationTest.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java index a75ad9b5ce..ba9a7b887e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/attributes/request/AbstractMqttAttributesRequestIntegrationTest.java @@ -66,7 +66,7 @@ public abstract class AbstractMqttAttributesRequestIntegrationTest extends Abstr postAttributesAndSubscribeToTopic(savedDevice, client); - Thread.sleep(1000); + Thread.sleep(2000); TestMqttCallback callback = getTestMqttCallback(); client.setCallback(callback); @@ -86,13 +86,13 @@ public abstract class AbstractMqttAttributesRequestIntegrationTest extends Abstr assertNotNull(savedDevice); - Thread.sleep(1000); + Thread.sleep(2000); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - Thread.sleep(1000); + Thread.sleep(2000); - client.subscribe(MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC, MqttQoS.AT_LEAST_ONCE.value()); + client.subscribe(MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC, MqttQoS.AT_LEAST_ONCE.value()).waitForCompletion(TimeUnit.MINUTES.toMillis(1)); TestMqttCallback clientAttributesCallback = getTestMqttCallback(); client.setCallback(clientAttributesCallback); @@ -105,13 +105,13 @@ public abstract class AbstractMqttAttributesRequestIntegrationTest extends Abstr protected void postAttributesAndSubscribeToTopic(Device savedDevice, MqttAsyncClient client) throws Exception { doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - client.publish(MqttTopics.DEVICE_ATTRIBUTES_TOPIC, new MqttMessage(POST_ATTRIBUTES_PAYLOAD.getBytes())); - client.subscribe(MqttTopics.DEVICE_ATTRIBUTES_RESPONSES_TOPIC, MqttQoS.AT_MOST_ONCE.value()); + client.publish(MqttTopics.DEVICE_ATTRIBUTES_TOPIC, new MqttMessage(POST_ATTRIBUTES_PAYLOAD.getBytes())).waitForCompletion(TimeUnit.MINUTES.toMillis(1)); + client.subscribe(MqttTopics.DEVICE_ATTRIBUTES_RESPONSES_TOPIC, MqttQoS.AT_MOST_ONCE.value()).waitForCompletion(TimeUnit.MINUTES.toMillis(1)); } protected void postGatewayDeviceClientAttributes(MqttAsyncClient client) throws Exception { String postClientAttributes = "{\"" + "Gateway Device Request Attributes" + "\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; - client.publish(MqttTopics.GATEWAY_ATTRIBUTES_TOPIC, new MqttMessage(postClientAttributes.getBytes())); + client.publish(MqttTopics.GATEWAY_ATTRIBUTES_TOPIC, new MqttMessage(postClientAttributes.getBytes())).waitForCompletion(TimeUnit.MINUTES.toMillis(1)); } protected void validateResponse(MqttAsyncClient client, CountDownLatch latch, TestMqttCallback callback) throws MqttException, InterruptedException, InvalidProtocolBufferException { @@ -119,8 +119,8 @@ public abstract class AbstractMqttAttributesRequestIntegrationTest extends Abstr String payloadStr = "{\"clientKeys\":\"" + keys + "\", \"sharedKeys\":\"" + keys + "\"}"; MqttMessage mqttMessage = new MqttMessage(); mqttMessage.setPayload(payloadStr.getBytes()); - client.publish(MqttTopics.DEVICE_ATTRIBUTES_REQUEST_TOPIC_PREFIX + "1", mqttMessage); - latch.await(3, TimeUnit.SECONDS); + client.publish(MqttTopics.DEVICE_ATTRIBUTES_REQUEST_TOPIC_PREFIX + "1", mqttMessage).waitForCompletion(TimeUnit.MINUTES.toMillis(1)); + latch.await(1, TimeUnit.MINUTES); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); String expectedRequestPayload = "{\"client\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}},\"shared\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(callback.getPayloadBytes(), StandardCharsets.UTF_8))); @@ -130,8 +130,8 @@ public abstract class AbstractMqttAttributesRequestIntegrationTest extends Abstr String payloadStr = "{\"id\": 1, \"device\": \"" + "Gateway Device Request Attributes" + "\", \"client\": true, \"keys\": [\"attribute1\", \"attribute2\", \"attribute3\", \"attribute4\", \"attribute5\"]}"; MqttMessage mqttMessage = new MqttMessage(); mqttMessage.setPayload(payloadStr.getBytes()); - client.publish(MqttTopics.GATEWAY_ATTRIBUTES_REQUEST_TOPIC, mqttMessage); - callback.getLatch().await(3, TimeUnit.SECONDS); + client.publish(MqttTopics.GATEWAY_ATTRIBUTES_REQUEST_TOPIC, mqttMessage).waitForCompletion(TimeUnit.MINUTES.toMillis(1)); + callback.getLatch().await(1, TimeUnit.MINUTES); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); String expectedRequestPayload = "{\"id\":1,\"device\":\"" + "Gateway Device Request Attributes" + "\",\"values\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(callback.getPayloadBytes(), StandardCharsets.UTF_8))); @@ -141,8 +141,8 @@ public abstract class AbstractMqttAttributesRequestIntegrationTest extends Abstr String payloadStr = "{\"id\": 1, \"device\": \"" + "Gateway Device Request Attributes" + "\", \"client\": false, \"keys\": [\"attribute1\", \"attribute2\", \"attribute3\", \"attribute4\", \"attribute5\"]}"; MqttMessage mqttMessage = new MqttMessage(); mqttMessage.setPayload(payloadStr.getBytes()); - client.publish(MqttTopics.GATEWAY_ATTRIBUTES_REQUEST_TOPIC, mqttMessage); - callback.getLatch().await(3, TimeUnit.SECONDS); + client.publish(MqttTopics.GATEWAY_ATTRIBUTES_REQUEST_TOPIC, mqttMessage).waitForCompletion(TimeUnit.MINUTES.toMillis(1)); + callback.getLatch().await(1, TimeUnit.MINUTES); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); String expectedRequestPayload = "{\"id\":1,\"device\":\"" + "Gateway Device Request Attributes" + "\",\"values\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(callback.getPayloadBytes(), StandardCharsets.UTF_8))); From 2700c429dda9574e12ea02a12bb50f754556e582 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Fri, 16 Jul 2021 19:25:08 +0300 Subject: [PATCH 9/9] Unique name for logs container in order to avoid container name conflicts --- .../thingsboard/server/msa/ThingsBoardDbInstaller.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 6308b603c3..e3b5fe2263 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.msa; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; @@ -136,13 +137,15 @@ public class ThingsBoardDbInstaller extends ExternalResource { File tbLogsDir = new File(targetDir); tbLogsDir.mkdirs(); - dockerCompose.withCommand("run -d --rm --name tb-logs-container -v " + volumeName + ":/root alpine tail -f /dev/null"); + String logsContainerName = "tb-logs-container-" + RandomStringUtils.randomAlphanumeric(10); + + dockerCompose.withCommand("run -d --rm --name " + logsContainerName + " -v " + volumeName + ":/root alpine tail -f /dev/null"); dockerCompose.invokeDocker(); - dockerCompose.withCommand("cp tb-logs-container:/root/. "+tbLogsDir.getAbsolutePath()); + dockerCompose.withCommand("cp " + logsContainerName + ":/root/. "+tbLogsDir.getAbsolutePath()); dockerCompose.invokeDocker(); - dockerCompose.withCommand("rm -f tb-logs-container"); + dockerCompose.withCommand("rm -f " + logsContainerName); dockerCompose.invokeDocker(); }