Merge pull request #6813 from smatvienko-tb/docker-compose-zookeeper-upgrade-3.8.0

[3.4] docker-compose minor fixes

docker/compose-utils.sh

@@ -26,7 +26,7 @@ function additionalComposeArgs() {
         ADDITIONAL_COMPOSE_ARGS="-f docker-compose.hybrid.yml"
         ;;
         *)
-        echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2
+        echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2
         exit 1
     esac
     echo $ADDITIONAL_COMPOSE_ARGS
@@ -55,7 +55,7 @@ function additionalComposeQueueArgs() {
         ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml"
         ;;
         *)
-        echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or confluent or aws-sqs or pubsub or rabbitmq or service-bus." >&2
+        echo "Unknown Queue service TB_QUEUE_TYPE value specified in the .env file: '${TB_QUEUE_TYPE}'. Should be either 'kafka' or 'confluent' or 'aws-sqs' or 'pubsub' or 'rabbitmq' or 'service-bus'." >&2
         exit 1
     esac
     echo $ADDITIONAL_COMPOSE_QUEUE_ARGS
@@ -85,7 +85,7 @@ function additionalComposeCacheArgs() {
         CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml"
         ;;
         *)
-        echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
+        echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2
         exit 1
     esac
     echo $CACHE_COMPOSE_ARGS
@@ -102,7 +102,7 @@ function additionalStartupServices() {
         ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra"
         ;;
         *)
-        echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2
+        echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2
         exit 1
     esac
 
@@ -115,7 +115,7 @@ function additionalStartupServices() {
         ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
         ;;
         *)
-        echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
+        echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2
         exit 1
     esac
 

docker/docker-compose.hybrid.yml

@@ -38,27 +38,23 @@ services:
     env_file:
       - tb-node.hybrid.env
     depends_on:
-      - zookeeper
       - postgres
       - cassandra
   tb-core2:
     env_file:
       - tb-node.hybrid.env
     depends_on:
-      - zookeeper
       - postgres
       - cassandra
   tb-rule-engine1:
     env_file:
       - tb-node.hybrid.env
     depends_on:
-      - zookeeper
       - postgres
       - cassandra
   tb-rule-engine2:
     env_file:
       - tb-node.hybrid.env
     depends_on:
-      - zookeeper
       - postgres
       - cassandra

docker/docker-compose.kafka.yml

@@ -95,4 +95,4 @@ services:
     env_file:
       - queue-kafka.env
     depends_on:
-      - kafka
+      - kafka

docker/docker-compose.pubsub.yml

@@ -59,4 +59,3 @@ services:
   tb-vc-executor2:
     env_file:
       - queue-pubsub.env
-

docker/docker-compose.rabbitmq.yml

@@ -58,4 +58,4 @@ services:
       - queue-rabbitmq.env
   tb-vc-executor2:
     env_file:
-      - queue-rabbitmq.env
+      - queue-rabbitmq.env

docker/docker-create-log-folders.sh

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-mkdir -p tb-node/log/ && sudo chown -R 799:799 tb-node/log/
+mkdir -p tb-node/log && sudo chown -R 799:799 tb-node/log
 
 mkdir -p tb-transports/coap/log && sudo chown -R 799:799 tb-transports/coap/log
 
@@ -29,23 +29,25 @@ mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log
 
 mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log
 
-mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/
+mkdir -p tb-node/postgres && sudo chown -R 999:999 tb-node/postgres
+
+mkdir -p tb-node/cassandra && sudo chown -R 999:999 tb-node/cassandra
 
 source .env
 CACHE="${CACHE:-redis}"
 case $CACHE in
     redis)
-    mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/
+    mkdir -p tb-node/redis-data && sudo chown -R 1001:1001 tb-node/redis-data
     ;;
     redis-cluster)
-    mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/
-    mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/
-    mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/
-    mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/
-    mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/
-    mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/
+    mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-0
+    mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-1
+    mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-2
+    mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-3
+    mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-4
+    mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-5
     ;;
     *)
-    echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
+    echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2
     exit 1
 esac

docker/docker-install-tb.sh

@@ -50,9 +50,14 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
 
 if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
-    docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
+    docker-compose \
+      -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \
+      up -d $ADDITIONAL_STARTUP_SERVICES
 fi
 
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \
+  run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \
+  tb-core1
 
 

docker/docker-remove-services.sh

@@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
 
 ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \
+  down -v

docker/docker-start-services.sh

@@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
 
 ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \
+  up -d

docker/docker-stop-services.sh

@@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
 
 ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \
+  stop

docker/docker-update-service.sh

@@ -25,5 +25,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
 
 ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \
+  pull $@
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \
+  up -d --no-deps --build $@

docker/docker-upgrade-tb.sh

@@ -48,8 +48,16 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
 
 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
 
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1
-
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
-
-docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \
+  pull \
+  tb-core1
+
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \
+  up -d $ADDITIONAL_STARTUP_SERVICES
+
+docker-compose \
+  -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \
+  run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \
+  tb-core1

msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@@ -77,7 +77,8 @@ public class ContainerTestSuite {
                     }
                 }
 
-                List<File> composeFiles = new ArrayList<>(Arrays.asList(new File(targetDir + "docker-compose.yml"),
+                List<File> composeFiles = new ArrayList<>(Arrays.asList(
+                        new File(targetDir + "docker-compose.yml"),
                         new File(targetDir + "docker-compose.volumes.yml"),
                         IS_HYBRID_MODE
                                 ? new File(targetDir + "docker-compose.hybrid.yml")
@@ -85,11 +86,11 @@ public class ContainerTestSuite {
                         new File(targetDir + "docker-compose.postgres.volumes.yml"),
                         new File(targetDir + "docker-compose.kafka.yml"),
                         IS_REDIS_CLUSTER
-                                ? new File("./../../docker/docker-compose.redis-cluster.yml")
-                                : new File("./../../docker/docker-compose.redis.yml"),
+                                ? new File(targetDir + "docker-compose.redis-cluster.yml")
+                                : new File(targetDir + "docker-compose.redis.yml"),
                         IS_REDIS_CLUSTER
-                                ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml")
-                                : new File("./../../docker/docker-compose.redis.volumes.yml")));
+                                ? new File(targetDir + "docker-compose.redis-cluster.volumes.yml")
+                                : new File(targetDir + "docker-compose.redis.volumes.yml")));
 
                 if (IS_HYBRID_MODE) {
                     composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml"));

msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java

@@ -21,7 +21,11 @@ import org.junit.rules.ExternalResource;
 import org.testcontainers.utility.Base58;
 
 import java.io.File;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;