From 40823dd2c0da480e3284a660644ff9fbcfb565f3 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 17:40:37 +0300 Subject: [PATCH 1/5] msa black box tests: redis-cluster test implemented with mvn -DblackBoxTests.redisCluster=true --- docker/cache-redis-cluster.env | 5 + docker/cache-redis.env | 2 + docker/docker-compose.hybrid.yml | 4 - docker/docker-compose.kafka.yml | 4 - docker/docker-compose.postgres.yml | 8 - docker/docker-compose.redis-cluster.yml | 162 ++++++++++++++++++ docker/docker-compose.redis.yml | 102 +++++++++++ docker/docker-compose.yml | 11 +- docker/docker-install-tb.sh | 4 +- docker/docker-remove-services.sh | 2 +- docker/docker-start-services.sh | 2 +- docker/docker-stop-services.sh | 2 +- docker/docker-update-service.sh | 4 +- docker/docker-upgrade-tb.sh | 6 +- docker/tb-coap-transport.env | 3 - docker/tb-http-transport.env | 3 - docker/tb-lwm2m-transport.env | 3 - docker/tb-mqtt-transport.env | 3 - docker/tb-node.env | 2 - docker/tb-snmp-transport.env | 3 - msa/black-box-tests/README.md | 7 +- .../server/msa/ContainerTestSuite.java | 8 +- .../server/msa/ThingsBoardDbInstaller.java | 46 ++++- 23 files changed, 337 insertions(+), 59 deletions(-) create mode 100644 docker/cache-redis-cluster.env create mode 100644 docker/cache-redis.env create mode 100644 docker/docker-compose.redis-cluster.yml create mode 100644 docker/docker-compose.redis.yml diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env new file mode 100644 index 0000000000..de666e003b --- /dev/null +++ b/docker/cache-redis-cluster.env @@ -0,0 +1,5 @@ +CACHE_TYPE=redis +REDIS_CONNECTION_TYPE=cluster +REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 +REDIS_USE_DEFAULT_POOL_CONFIG=false +REDIS_PASSWORD=bitnami \ No newline at end of file diff --git a/docker/cache-redis.env b/docker/cache-redis.env new file mode 100644 index 0000000000..52091ecc31 --- /dev/null +++ b/docker/cache-redis.env @@ -0,0 +1,2 @@ +CACHE_TYPE=redis +REDIS_HOST=redis \ No newline at end of file diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 7d7ab84c8f..e3ba32b779 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -39,7 +39,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-core2: @@ -47,7 +46,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-rule-engine1: @@ -55,7 +53,6 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra tb-rule-engine2: @@ -63,6 +60,5 @@ services: - tb-node.hybrid.env depends_on: - zookeeper - - redis - postgres - cassandra diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 1683114605..82fbb6ed62 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -36,25 +36,21 @@ services: - queue-kafka.env depends_on: - kafka - - redis tb-core2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine1: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-rule-engine2: env_file: - queue-kafka.env depends_on: - kafka - - redis tb-mqtt-transport1: env_file: - queue-kafka.env diff --git a/docker/docker-compose.postgres.yml b/docker/docker-compose.postgres.yml index 591ea59f7a..8fe8e6f53d 100644 --- a/docker/docker-compose.postgres.yml +++ b/docker/docker-compose.postgres.yml @@ -31,27 +31,19 @@ services: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-core2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine1: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres tb-rule-engine2: env_file: - tb-node.postgres.env depends_on: - - zookeeper - - redis - postgres diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml new file mode 100644 index 0000000000..e66122f44d --- /dev/null +++ b/docker/docker-compose.redis-cluster.yml @@ -0,0 +1,162 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis cluster + redis-node-0: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-0:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-1: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-1:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-2: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-2:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-3: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-3:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-4: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-4:/bitnami/redis/data + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + + redis-node-5: + image: bitnami/redis-cluster:7.0 + volumes: + - redis-cluster_data-5:/bitnami/redis/data + depends_on: + - redis-node-0 + - redis-node-1 + - redis-node-2 + - redis-node-3 + - redis-node-4 + environment: + - 'REDIS_PASSWORD=bitnami' + - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' + - 'REDISCLI_AUTH=bitnami' + - 'REDIS_CLUSTER_REPLICAS=1' + - 'REDIS_CLUSTER_CREATOR=yes' + +# ThingsBoard setup to use redis-cluster + tb-core1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-core2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-rule-engine2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-mqtt-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-http-transport2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-coap-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-lwm2m-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-snmp-transport: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor1: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 + tb-vc-executor2: + env_file: + - cache-redis-cluster.env + depends_on: + - redis-node-5 +volumes: + redis-cluster_data-0: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_0} + redis-cluster_data-1: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_1} + redis-cluster_data-2: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_2} + redis-cluster_data-3: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_3} + redis-cluster_data-4: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_4} + redis-cluster_data-5: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml new file mode 100644 index 0000000000..671c1952b8 --- /dev/null +++ b/docker/docker-compose.redis.yml @@ -0,0 +1,102 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: +# Redis standalone + redis: + restart: always + image: bitnami/redis:7.0 + environment: + # ALLOW_EMPTY_PASSWORD is recommended only for development. + ALLOW_EMPTY_PASSWORD: "yes" + ports: + - '6379:6379' + volumes: + - 'redis_data:/bitnami/redis/data' + +# ThingsBoard setup to use redis-standalone + tb-core1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-core2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-rule-engine2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-mqtt-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-http-transport2: + env_file: + - cache-redis.env + depends_on: + - redis + tb-coap-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-lwm2m-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-snmp-transport: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor1: + env_file: + - cache-redis.env + depends_on: + - redis + tb-vc-executor2: + env_file: + - cache-redis.env + depends_on: + - redis + +volumes: + redis_data: + external: true + name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1ba6eda32d..62969b1051 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -26,15 +26,10 @@ services: environment: ZOO_MY_ID: 1 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 - redis: - restart: always - image: redis:4.0 - ports: - - "6379" tb-js-executor: restart: always image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" - scale: 20 + scale: 10 env_file: - tb-js-executor.env tb-core1: @@ -59,7 +54,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -85,7 +79,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor - tb-rule-engine1 - tb-rule-engine2 @@ -109,7 +102,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-rule-engine2: restart: always @@ -131,7 +123,6 @@ services: - ./tb-node/log:/var/log/thingsboard depends_on: - zookeeper - - redis - tb-js-executor tb-mqtt-transport1: restart: always diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 6f6b1e9511..3ab5a873f0 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -48,9 +48,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 89f0f7844c..11bd451aa1 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 9f159774d8..e6b06718c5 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 61e68d6dd5..dd2ccf9ef9 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -25,4 +25,4 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 739fcf6543..e6ec0c62a4 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -23,5 +23,5 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 3b1415a965..6a4311ffba 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -46,8 +46,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 diff --git a/docker/tb-coap-transport.env b/docker/tb-coap-transport.env index 9e6a41c930..079443c98b 100644 --- a/docker/tb-coap-transport.env +++ b/docker/tb-coap-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-http-transport.env b/docker/tb-http-transport.env index 1b4ce7a298..7e0679987f 100644 --- a/docker/tb-http-transport.env +++ b/docker/tb-http-transport.env @@ -7,6 +7,3 @@ HTTP_REQUEST_TIMEOUT=60000 METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-lwm2m-transport.env b/docker/tb-lwm2m-transport.env index 4616d45972..f284803a46 100644 --- a/docker/tb-lwm2m-transport.env +++ b/docker/tb-lwm2m-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-mqtt-transport.env b/docker/tb-mqtt-transport.env index 0cd51c7371..e38cb2124a 100644 --- a/docker/tb-mqtt-transport.env +++ b/docker/tb-mqtt-transport.env @@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/docker/tb-node.env b/docker/tb-node.env index e3393d41b1..ba66757ecc 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -4,8 +4,6 @@ ZOOKEEPER_ENABLED=true ZOOKEEPER_URL=zookeeper:2181 JS_EVALUATOR=remote TRANSPORT_TYPE=remote -CACHE_TYPE=redis -REDIS_HOST=redis HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false diff --git a/docker/tb-snmp-transport.env b/docker/tb-snmp-transport.env index 4851e9f6c1..e2cc39d658 100644 --- a/docker/tb-snmp-transport.env +++ b/docker/tb-snmp-transport.env @@ -6,6 +6,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus WEB_APPLICATION_ENABLE=true WEB_APPLICATION_TYPE=servlet HTTP_BIND_PORT=8081 - -CACHE_TYPE=redis -REDIS_HOST=redis diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index 9c68789e67..6e70de17ca 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -18,8 +18,13 @@ As result, in REPOSITORY column, next images should be present: thingsboard/tb-web-ui thingsboard/tb-js-executor -- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory: +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone: mvn clean install -DblackBoxTests.skip=false +- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis cluster: + + mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index f60d6d7545..c9ed8a4040 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -39,7 +39,7 @@ import static org.junit.Assert.fail; @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) @Slf4j public class ContainerTestSuite { - + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; @@ -52,6 +52,7 @@ public class ContainerTestSuite { @ClassRule public static DockerComposeContainer getTestContainer() { if (testContainer == null) { + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); try { final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; @@ -75,7 +76,10 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose.kafka.yml")) + new File(targetDir + "docker-compose.kafka.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.yml") + : new File("./../../docker/docker-compose.redis.yml")) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index de55a3afa3..ac0fa55991 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.msa; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; @@ -24,10 +25,16 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +@Slf4j public class ThingsBoardDbInstaller extends ExternalResource { + final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; + private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume"; + private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume"; private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume"; private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume"; @@ -39,6 +46,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final DockerComposeExecutor dockerCompose; private final String postgresDataVolume; + + private final String redisDataVolume; + private final String redisClusterDataVolume; private final String tbLogVolume; private final String tbCoapTransportLogVolume; private final String tbLwm2mTransportLogVolume; @@ -49,14 +59,21 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final Map env; public ThingsBoardDbInstaller() { + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), new File("./../../docker/docker-compose.postgres.yml"), - new File("./../../docker/docker-compose.postgres.volumes.yml")); + new File("./../../docker/docker-compose.postgres.volumes.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.yml") + : new File("./../../docker/docker-compose.redis.yml") + ); String identifier = Base58.randomString(6).toLowerCase(); String project = identifier + Base58.randomString(6).toLowerCase(); postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; + redisDataVolume = project + "_" + REDIS_DATA_VOLUME; + redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME; tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME; tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME; @@ -76,6 +93,13 @@ public class ThingsBoardDbInstaller extends ExternalResource { env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); + } + } else { + env.put("REDIS_DATA_VOLUME", redisDataVolume); + } dockerCompose.withEnv(env); } @@ -111,7 +135,20 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); - dockerCompose.withCommand("up -d redis postgres"); + String redisService = ""; + if (IS_REDIS_CLUSTER) { + for (int i = 0; i < 6; i++) { + redisService = redisService + " redis-node-" + i; + dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i); + dockerCompose.invokeDocker(); + } + } else { + redisService = "redis"; + dockerCompose.withCommand("volume create " + redisDataVolume); + dockerCompose.invokeDocker(); + } + + dockerCompose.withCommand("up -d postgres " + redisService); dockerCompose.invokeCompose(); dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); @@ -137,7 +174,10 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + - " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); + " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume + + (IS_REDIS_CLUSTER + ? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining()) + : redisDataVolume)); dockerCompose.invokeDocker(); } From d7c750f64e60a1dd06aa9436b39cbe9fabbecd13 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 21:30:34 +0300 Subject: [PATCH 2/5] msa black box tests: redis, postgres, node volumes refactored - volumes decoupled to separate files .volumes (used only for tests) --- docker/cache-redis-cluster.env | 2 +- docker/cache-redis.env | 2 +- docker/docker-compose.postgres.volumes.yml | 61 -------------- .../docker-compose.redis-cluster.volumes.yml | 58 +++++++++++++ docker/docker-compose.redis-cluster.yml | 31 ++----- docker/docker-compose.redis.volumes.yml | 27 +++++++ docker/docker-compose.redis.yml | 7 +- docker/docker-compose.volumes.yml | 81 +++++++++++++++++++ .../server/msa/ContainerTestSuite.java | 7 +- .../server/msa/ThingsBoardDbInstaller.java | 9 ++- 10 files changed, 188 insertions(+), 97 deletions(-) create mode 100644 docker/docker-compose.redis-cluster.volumes.yml create mode 100644 docker/docker-compose.redis.volumes.yml create mode 100644 docker/docker-compose.volumes.yml diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env index de666e003b..64058ab556 100644 --- a/docker/cache-redis-cluster.env +++ b/docker/cache-redis-cluster.env @@ -2,4 +2,4 @@ CACHE_TYPE=redis REDIS_CONNECTION_TYPE=cluster REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 REDIS_USE_DEFAULT_POOL_CONFIG=false -REDIS_PASSWORD=bitnami \ No newline at end of file +REDIS_PASSWORD=bitnami diff --git a/docker/cache-redis.env b/docker/cache-redis.env index 52091ecc31..7b92620666 100644 --- a/docker/cache-redis.env +++ b/docker/cache-redis.env @@ -1,2 +1,2 @@ CACHE_TYPE=redis -REDIS_HOST=redis \ No newline at end of file +REDIS_HOST=redis diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml index 019e087c48..caf78f23d7 100644 --- a/docker/docker-compose.postgres.volumes.yml +++ b/docker/docker-compose.postgres.volumes.yml @@ -20,69 +20,8 @@ services: postgres: volumes: - postgres-db-volume:/var/lib/postgresql/data - tb-core1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-core2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine1: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-rule-engine2: - volumes: - - tb-log-volume:/var/log/thingsboard - tb-coap-transport: - volumes: - - tb-coap-transport-log-volume:/var/log/tb-coap-transport - tb-lwm2m-transport: - volumes: - - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport - tb-http-transport1: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-http-transport2: - volumes: - - tb-http-transport-log-volume:/var/log/tb-http-transport - tb-mqtt-transport1: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-mqtt-transport2: - volumes: - - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport - tb-snmp-transport: - volumes: - - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport - tb-vc-executor1: - volumes: - - tb-vc-executor-log-volume:/var/log/tb-vc-executor - tb-vc-executor2: - volumes: - - tb-vc-executor-log-volume:/var/log/tb-vc-executor - volumes: postgres-db-volume: external: true name: ${POSTGRES_DATA_VOLUME} - tb-log-volume: - external: true - name: ${TB_LOG_VOLUME} - tb-coap-transport-log-volume: - external: true - name: ${TB_COAP_TRANSPORT_LOG_VOLUME} - tb-lwm2m-transport-log-volume: - external: true - name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} - tb-http-transport-log-volume: - external: true - name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} - tb-mqtt-transport-log-volume: - external: true - name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} - tb-snmp-transport-log-volume: - external: true - name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} - tb-vc-executor-log-volume: - external: true - name: ${TB_VC_EXECUTOR_LOG_VOLUME} \ No newline at end of file diff --git a/docker/docker-compose.redis-cluster.volumes.yml b/docker/docker-compose.redis-cluster.volumes.yml new file mode 100644 index 0000000000..2cf319bd21 --- /dev/null +++ b/docker/docker-compose.redis-cluster.volumes.yml @@ -0,0 +1,58 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + # Redis cluster + redis-node-0: + volumes: + - redis-cluster-data-0:/bitnami/redis/data + redis-node-1: + volumes: + - redis-cluster-data-1:/bitnami/redis/data + redis-node-2: + volumes: + - redis-cluster-data-2:/bitnami/redis/data + redis-node-3: + volumes: + - redis-cluster-data-3:/bitnami/redis/data + redis-node-4: + volumes: + - redis-cluster-data-4:/bitnami/redis/data + redis-node-5: + volumes: + - redis-cluster-data-5:/bitnami/redis/data + +volumes: + redis-cluster-data-0: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_0} + redis-cluster-data-1: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_1} + redis-cluster-data-2: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_2} + redis-cluster-data-3: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_3} + redis-cluster-data-4: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_4} + redis-cluster-data-5: + external: true + name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index e66122f44d..17ca4ed13a 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -21,7 +21,7 @@ services: redis-node-0: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-0:/bitnami/redis/data + - ./tb-node/redis-cluster-data-0:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -29,7 +29,7 @@ services: redis-node-1: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-1:/bitnami/redis/data + - ./tb-node/redis-cluster-data-1:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -37,7 +37,7 @@ services: redis-node-2: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-2:/bitnami/redis/data + - ./tb-node/redis-cluster-data-2:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -45,7 +45,7 @@ services: redis-node-3: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-3:/bitnami/redis/data + - ./tb-node/redis-cluster-data-3:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -53,7 +53,7 @@ services: redis-node-4: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-4:/bitnami/redis/data + - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - 'REDIS_PASSWORD=bitnami' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' @@ -61,7 +61,7 @@ services: redis-node-5: image: bitnami/redis-cluster:7.0 volumes: - - redis-cluster_data-5:/bitnami/redis/data + - ./tb-node/redis-cluster-data-5:/bitnami/redis/data depends_on: - redis-node-0 - redis-node-1 @@ -141,22 +141,3 @@ services: - cache-redis-cluster.env depends_on: - redis-node-5 -volumes: - redis-cluster_data-0: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_0} - redis-cluster_data-1: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_1} - redis-cluster_data-2: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_2} - redis-cluster_data-3: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_3} - redis-cluster_data-4: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_4} - redis-cluster_data-5: - external: true - name: ${REDIS_CLUSTER_DATA_VOLUME_5} diff --git a/docker/docker-compose.redis.volumes.yml b/docker/docker-compose.redis.volumes.yml new file mode 100644 index 0000000000..090aa441fe --- /dev/null +++ b/docker/docker-compose.redis.volumes.yml @@ -0,0 +1,27 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + redis: + volumes: + - redis-data:/bitnami/redis/data + +volumes: + redis-data: + external: true + name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.redis.yml b/docker/docker-compose.redis.yml index 671c1952b8..e53a974134 100644 --- a/docker/docker-compose.redis.yml +++ b/docker/docker-compose.redis.yml @@ -27,7 +27,7 @@ services: ports: - '6379:6379' volumes: - - 'redis_data:/bitnami/redis/data' + - ./tb-node/redis-data:/bitnami/redis/data # ThingsBoard setup to use redis-standalone tb-core1: @@ -95,8 +95,3 @@ services: - cache-redis.env depends_on: - redis - -volumes: - redis_data: - external: true - name: ${REDIS_DATA_VOLUME} diff --git a/docker/docker-compose.volumes.yml b/docker/docker-compose.volumes.yml new file mode 100644 index 0000000000..58269473e4 --- /dev/null +++ b/docker/docker-compose.volumes.yml @@ -0,0 +1,81 @@ +# +# Copyright © 2016-2022 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + tb-core1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-core2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-rule-engine2: + volumes: + - tb-log-volume:/var/log/thingsboard + tb-coap-transport: + volumes: + - tb-coap-transport-log-volume:/var/log/tb-coap-transport + tb-lwm2m-transport: + volumes: + - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport + tb-http-transport1: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-http-transport2: + volumes: + - tb-http-transport-log-volume:/var/log/tb-http-transport + tb-mqtt-transport1: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-mqtt-transport2: + volumes: + - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport + tb-snmp-transport: + volumes: + - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport + tb-vc-executor1: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + tb-vc-executor2: + volumes: + - tb-vc-executor-log-volume:/var/log/tb-vc-executor + +volumes: + tb-log-volume: + external: true + name: ${TB_LOG_VOLUME} + tb-coap-transport-log-volume: + external: true + name: ${TB_COAP_TRANSPORT_LOG_VOLUME} + tb-lwm2m-transport-log-volume: + external: true + name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME} + tb-http-transport-log-volume: + external: true + name: ${TB_HTTP_TRANSPORT_LOG_VOLUME} + tb-mqtt-transport-log-volume: + external: true + name: ${TB_MQTT_TRANSPORT_LOG_VOLUME} + tb-snmp-transport-log-volume: + external: true + name: ${TB_SNMP_TRANSPORT_LOG_VOLUME} + tb-vc-executor-log-volume: + external: true + name: ${TB_VC_EXECUTOR_LOG_VOLUME} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index c9ed8a4040..cc6661d8d9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -74,12 +74,17 @@ public class ContainerTestSuite { testContainer = new DockerComposeContainerImpl<>( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml")) + : new File("./../../docker/docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") + : new File("./../../docker/docker-compose.redis.volumes.yml") + ) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index ac0fa55991..08a333ab52 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -60,12 +60,17 @@ public class ThingsBoardDbInstaller extends ExternalResource { public ThingsBoardDbInstaller() { log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); - List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), + List composeFiles = Arrays.asList( + new File("./../../docker/docker-compose.yml"), + new File("./../../docker/docker-compose.volumes.yml"), new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.volumes.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml") + : new File("./../../docker/docker-compose.redis.yml"), + IS_REDIS_CLUSTER + ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") + : new File("./../../docker/docker-compose.redis.volumes.yml") ); String identifier = Base58.randomString(6).toLowerCase(); From 7b88a437dc194eaab7dc0f2aeebf7dd6d45e2707 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 22 Jun 2022 23:24:05 +0300 Subject: [PATCH 3/5] docker-compose shell scripts: added CACHE=redis or CACHE=redis-cluster option. Default is redis, README.md updated --- docker/.env | 3 +++ docker/README.md | 7 ++++++ docker/compose-utils.sh | 36 +++++++++++++++++++++++++++-- docker/docker-create-log-folders.sh | 21 +++++++++++++++++ docker/docker-install-tb.sh | 6 +++-- docker/docker-remove-services.sh | 4 +++- docker/docker-start-services.sh | 4 +++- docker/docker-stop-services.sh | 4 +++- docker/docker-update-service.sh | 6 +++-- docker/docker-upgrade-tb.sh | 8 ++++--- 10 files changed, 87 insertions(+), 12 deletions(-) diff --git a/docker/.env b/docker/.env index af0f537603..7bc1a7a6a4 100644 --- a/docker/.env +++ b/docker/.env @@ -1,5 +1,8 @@ TB_QUEUE_TYPE=kafka +# redis or redis-cluster +CACHE=redis + DOCKER_REPO=thingsboard JS_EXECUTOR_DOCKER_NAME=tb-js-executor diff --git a/docker/README.md b/docker/README.md index 01f89ebb6c..71ea87f353 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env` **NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details). +In order to set cache type change the value of `CACHE` variable in `.env` file to one of the following: + +- `redis` - use Redis standalone cache (1 node - 1 master); +- `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves); + +**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details). + Execute the following command to create log folders for the services and chown of these folders to the docker container users. To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access): diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index 550f2dfea4..d60c81ec6a 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -73,19 +73,51 @@ function additionalComposeMonitoringArgs() { fi } +function additionalComposeCacheArgs() { + source .env + CACHE_COMPOSE_ARGS="" + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + CACHE_COMPOSE_ARGS="-f docker-compose.redis.yml" + ;; + redis-cluster) + CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 + esac + echo $CACHE_COMPOSE_ARGS +} + function additionalStartupServices() { source .env ADDITIONAL_STARTUP_SERVICES="" case $DATABASE in postgres) - ADDITIONAL_STARTUP_SERVICES=postgres + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres" ;; hybrid) - ADDITIONAL_STARTUP_SERVICES="postgres cassandra" + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra" ;; *) echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 exit 1 esac + + CACHE="${CACHE:-redis}" + case $CACHE in + redis) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis" + ;; + redis-cluster) + ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 + esac + echo $ADDITIONAL_STARTUP_SERVICES } diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index ba945a19df..83257c0e66 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -28,3 +28,24 @@ mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log + +mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/ + +source .env +CACHE="${CACHE:-redis}" +case $CACHE in + redis) + mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/ + ;; + redis-cluster) + mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/ + mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/ + mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/ + mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/ + mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/ + mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/ + ;; + *) + echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + exit 1 +esac \ No newline at end of file diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 3ab5a873f0..27e84c4d0b 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -45,12 +45,14 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES + docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 11bd451aa1..36e464f45d 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index e6b06718c5..7993ab425b 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index dd2ccf9ef9..ae8a3c0b81 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index e6ec0c62a4..a02f9e208a 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -23,5 +23,7 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 6a4311ffba..c20543783b 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -44,10 +44,12 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? +ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES -docker-compose -f docker-compose.yml docker-compose.redis.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 From d903d0ddc0db3a6797552145a5aca4f4bed7e43e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 12:39:50 +0300 Subject: [PATCH 4/5] default Redis cluster password changed, set REDISCLI_AUTH to simplify redis-cli usage with docker exec --- docker/cache-redis-cluster.env | 2 +- docker/docker-compose.redis-cluster.yml | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docker/cache-redis-cluster.env b/docker/cache-redis-cluster.env index 64058ab556..a3b516063b 100644 --- a/docker/cache-redis-cluster.env +++ b/docker/cache-redis-cluster.env @@ -2,4 +2,4 @@ CACHE_TYPE=redis REDIS_CONNECTION_TYPE=cluster REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379 REDIS_USE_DEFAULT_POOL_CONFIG=false -REDIS_PASSWORD=bitnami +REDIS_PASSWORD=thingsboard diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index 17ca4ed13a..0a11b765e8 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -23,7 +23,8 @@ services: volumes: - ./tb-node/redis-cluster-data-0:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-1: @@ -31,7 +32,8 @@ services: volumes: - ./tb-node/redis-cluster-data-1:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-2: @@ -39,7 +41,8 @@ services: volumes: - ./tb-node/redis-cluster-data-2:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-3: @@ -47,7 +50,8 @@ services: volumes: - ./tb-node/redis-cluster-data-3:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-4: @@ -55,7 +59,7 @@ services: volumes: - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-5: @@ -69,9 +73,9 @@ services: - redis-node-3 - redis-node-4 environment: - - 'REDIS_PASSWORD=bitnami' + - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' - - 'REDISCLI_AUTH=bitnami' - 'REDIS_CLUSTER_REPLICAS=1' - 'REDIS_CLUSTER_CREATOR=yes' From 59b04e6b56b9c957f44d7683b02eb08c1279983e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 12:42:17 +0300 Subject: [PATCH 5/5] REDISCLI_AUTH password --- docker/docker-compose.redis-cluster.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-compose.redis-cluster.yml b/docker/docker-compose.redis-cluster.yml index 0a11b765e8..55c1c3f1db 100644 --- a/docker/docker-compose.redis-cluster.yml +++ b/docker/docker-compose.redis-cluster.yml @@ -60,6 +60,7 @@ services: - ./tb-node/redis-cluster-data-4:/bitnami/redis/data environment: - 'REDIS_PASSWORD=thingsboard' + - 'REDISCLI_AUTH=thingsboard' - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5' redis-node-5: