Merge pull request #6793 from smatvienko-tb/msa-black-box-tests-redis-cluster

[3.4] msa black box tests: redis-cluster test implemented …
4 years ago · 5fb462eea6
31 changed files with 572 additions and 123 deletions
--- a/docker/.env
+++ b/docker/.env
@ -1,5 +1,8 @@
 TB_QUEUE_TYPE=kafka

+# redis or redis-cluster
+CACHE=redis
+
 DOCKER_REPO=thingsboard

 JS_EXECUTOR_DOCKER_NAME=tb-js-executor
--- a/docker/README.md
+++ b/docker/README.md
@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env`

 **NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details).

+In order to set cache type change the value of `CACHE` variable in `.env` file to one of the following:
+
+- `redis` - use Redis standalone cache (1 node - 1 master);
+- `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves);
+
+**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details).
+
 Execute the following command to create log folders for the services and chown of these folders to the docker container users. 
 To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access): 

--- a/docker/cache-redis-cluster.env
+++ b/docker/cache-redis-cluster.env
@ -0,0 +1,5 @@
+CACHE_TYPE=redis
+REDIS_CONNECTION_TYPE=cluster
+REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379
+REDIS_USE_DEFAULT_POOL_CONFIG=false
+REDIS_PASSWORD=thingsboard
--- a/docker/cache-redis.env
+++ b/docker/cache-redis.env
@ -0,0 +1,2 @@
+CACHE_TYPE=redis
+REDIS_HOST=redis
--- a/docker/compose-utils.sh
+++ b/docker/compose-utils.sh
@ -73,19 +73,51 @@ function additionalComposeMonitoringArgs() {
    fi
 }

+function additionalComposeCacheArgs() {
+    source .env
+    CACHE_COMPOSE_ARGS=""
+    CACHE="${CACHE:-redis}"
+    case $CACHE in
+        redis)
+        CACHE_COMPOSE_ARGS="-f docker-compose.redis.yml"
+        ;;
+        redis-cluster)
+        CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml"
+        ;;
+        *)
+        echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
+        exit 1
+    esac
+    echo $CACHE_COMPOSE_ARGS
+}
+
 function additionalStartupServices() {
    source .env
    ADDITIONAL_STARTUP_SERVICES=""
    case $DATABASE in
        postgres)
-        ADDITIONAL_STARTUP_SERVICES=postgres
+        ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres"
        ;;
        hybrid)
-        ADDITIONAL_STARTUP_SERVICES="postgres cassandra"
+        ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra"
        ;;
        *)
        echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2
        exit 1
    esac
+
+    CACHE="${CACHE:-redis}"
+    case $CACHE in
+        redis)
+        ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis"
+        ;;
+        redis-cluster)
+        ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
+        ;;
+        *)
+        echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
+        exit 1
+    esac
+
    echo $ADDITIONAL_STARTUP_SERVICES
 }
--- a/docker/docker-compose.hybrid.yml
+++ b/docker/docker-compose.hybrid.yml
@ -39,7 +39,6 @@ services:
      - tb-node.hybrid.env
    depends_on:
      - zookeeper
-      - redis
      - postgres
      - cassandra
  tb-core2:
@ -47,7 +46,6 @@ services:
      - tb-node.hybrid.env
    depends_on:
      - zookeeper
-      - redis
      - postgres
      - cassandra
  tb-rule-engine1:
@ -55,7 +53,6 @@ services:
      - tb-node.hybrid.env
    depends_on:
      - zookeeper
-      - redis
      - postgres
      - cassandra
  tb-rule-engine2:
@ -63,6 +60,5 @@ services:
      - tb-node.hybrid.env
    depends_on:
      - zookeeper
-      - redis
      - postgres
      - cassandra
--- a/docker/docker-compose.kafka.yml
+++ b/docker/docker-compose.kafka.yml
@ -36,25 +36,21 @@ services:
      - queue-kafka.env
    depends_on:
      - kafka
-      - redis
  tb-core2:
    env_file:
      - queue-kafka.env
    depends_on:
      - kafka
-      - redis
  tb-rule-engine1:
    env_file:
      - queue-kafka.env
    depends_on:
      - kafka
-      - redis
  tb-rule-engine2:
    env_file:
      - queue-kafka.env
    depends_on:
      - kafka
-      - redis
  tb-mqtt-transport1:
    env_file:
      - queue-kafka.env
--- a/docker/docker-compose.postgres.volumes.yml
+++ b/docker/docker-compose.postgres.volumes.yml
@ -20,69 +20,8 @@ services:
  postgres:
    volumes:
      - postgres-db-volume:/var/lib/postgresql/data
-  tb-core1:
-    volumes:
-      - tb-log-volume:/var/log/thingsboard
-  tb-core2:
-    volumes:
-      - tb-log-volume:/var/log/thingsboard
-  tb-rule-engine1:
-    volumes:
-      - tb-log-volume:/var/log/thingsboard
-  tb-rule-engine2:
-    volumes:
-      - tb-log-volume:/var/log/thingsboard
-  tb-coap-transport:
-    volumes:
-      - tb-coap-transport-log-volume:/var/log/tb-coap-transport
-  tb-lwm2m-transport:
-    volumes:
-      - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport
-  tb-http-transport1:
-    volumes:
-      - tb-http-transport-log-volume:/var/log/tb-http-transport
-  tb-http-transport2:
-    volumes:
-      - tb-http-transport-log-volume:/var/log/tb-http-transport
-  tb-mqtt-transport1:
-    volumes:
-      - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
-  tb-mqtt-transport2:
-    volumes:
-      - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
-  tb-snmp-transport:
-    volumes:
-      - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport
-  tb-vc-executor1:
-    volumes:
-      - tb-vc-executor-log-volume:/var/log/tb-vc-executor
-  tb-vc-executor2:
-    volumes:
-      - tb-vc-executor-log-volume:/var/log/tb-vc-executor
-

 volumes:
  postgres-db-volume:
    external: true
    name: ${POSTGRES_DATA_VOLUME}
-  tb-log-volume:
-    external: true
-    name: ${TB_LOG_VOLUME}
-  tb-coap-transport-log-volume:
-    external: true
-    name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
-  tb-lwm2m-transport-log-volume:
-    external: true
-    name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME}
-  tb-http-transport-log-volume:
-    external: true
-    name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
-  tb-mqtt-transport-log-volume:
-    external: true
-    name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}
-  tb-snmp-transport-log-volume:
-    external: true
-    name: ${TB_SNMP_TRANSPORT_LOG_VOLUME}
-  tb-vc-executor-log-volume:
-    external: true
-    name: ${TB_VC_EXECUTOR_LOG_VOLUME}
--- a/docker/docker-compose.postgres.yml
+++ b/docker/docker-compose.postgres.yml
@ -31,27 +31,19 @@ services:
    env_file:
      - tb-node.postgres.env
    depends_on:
-      - zookeeper
-      - redis
      - postgres
  tb-core2:
    env_file:
      - tb-node.postgres.env
    depends_on:
-      - zookeeper
-      - redis
      - postgres
  tb-rule-engine1:
    env_file:
      - tb-node.postgres.env
    depends_on:
-      - zookeeper
-      - redis
      - postgres
  tb-rule-engine2:
    env_file:
      - tb-node.postgres.env
    depends_on:
-      - zookeeper
-      - redis
      - postgres
--- a/docker/docker-compose.redis-cluster.volumes.yml
+++ b/docker/docker-compose.redis-cluster.volumes.yml
@ -0,0 +1,58 @@
+#
+# Copyright © 2016-2022 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+services:
+  # Redis cluster
+  redis-node-0:
+    volumes:
+      - redis-cluster-data-0:/bitnami/redis/data
+  redis-node-1:
+    volumes:
+      - redis-cluster-data-1:/bitnami/redis/data
+  redis-node-2:
+    volumes:
+      - redis-cluster-data-2:/bitnami/redis/data
+  redis-node-3:
+    volumes:
+      - redis-cluster-data-3:/bitnami/redis/data
+  redis-node-4:
+    volumes:
+      - redis-cluster-data-4:/bitnami/redis/data
+  redis-node-5:
+    volumes:
+      - redis-cluster-data-5:/bitnami/redis/data
+
+volumes:
+  redis-cluster-data-0:
+    external: true
+    name: ${REDIS_CLUSTER_DATA_VOLUME_0}
+  redis-cluster-data-1:
+    external: true
+    name: ${REDIS_CLUSTER_DATA_VOLUME_1}
+  redis-cluster-data-2:
+    external: true
+    name: ${REDIS_CLUSTER_DATA_VOLUME_2}
+  redis-cluster-data-3:
+    external: true
+    name: ${REDIS_CLUSTER_DATA_VOLUME_3}
+  redis-cluster-data-4:
+    external: true
+    name: ${REDIS_CLUSTER_DATA_VOLUME_4}
+  redis-cluster-data-5:
+    external: true
+    name: ${REDIS_CLUSTER_DATA_VOLUME_5}
--- a/docker/docker-compose.redis-cluster.yml
+++ b/docker/docker-compose.redis-cluster.yml
@ -0,0 +1,148 @@
+#
+# Copyright © 2016-2022 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+services:
+# Redis cluster
+  redis-node-0:
+    image: bitnami/redis-cluster:7.0
+    volumes:
+      - ./tb-node/redis-cluster-data-0:/bitnami/redis/data
+    environment:
+      - 'REDIS_PASSWORD=thingsboard'
+      - 'REDISCLI_AUTH=thingsboard'
+      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+
+  redis-node-1:
+    image: bitnami/redis-cluster:7.0
+    volumes:
+      - ./tb-node/redis-cluster-data-1:/bitnami/redis/data
+    environment:
+      - 'REDIS_PASSWORD=thingsboard'
+      - 'REDISCLI_AUTH=thingsboard'
+      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+
+  redis-node-2:
+    image: bitnami/redis-cluster:7.0
+    volumes:
+      - ./tb-node/redis-cluster-data-2:/bitnami/redis/data
+    environment:
+      - 'REDIS_PASSWORD=thingsboard'
+      - 'REDISCLI_AUTH=thingsboard'
+      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+
+  redis-node-3:
+    image: bitnami/redis-cluster:7.0
+    volumes:
+      - ./tb-node/redis-cluster-data-3:/bitnami/redis/data
+    environment:
+      - 'REDIS_PASSWORD=thingsboard'
+      - 'REDISCLI_AUTH=thingsboard'
+      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+
+  redis-node-4:
+    image: bitnami/redis-cluster:7.0
+    volumes:
+      - ./tb-node/redis-cluster-data-4:/bitnami/redis/data
+    environment:
+      - 'REDIS_PASSWORD=thingsboard'
+      - 'REDISCLI_AUTH=thingsboard'
+      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+
+  redis-node-5:
+    image: bitnami/redis-cluster:7.0
+    volumes:
+      - ./tb-node/redis-cluster-data-5:/bitnami/redis/data
+    depends_on:
+      - redis-node-0
+      - redis-node-1
+      - redis-node-2
+      - redis-node-3
+      - redis-node-4
+    environment:
+      - 'REDIS_PASSWORD=thingsboard'
+      - 'REDISCLI_AUTH=thingsboard'
+      - 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
+      - 'REDIS_CLUSTER_REPLICAS=1'
+      - 'REDIS_CLUSTER_CREATOR=yes'
+
+# ThingsBoard setup to use redis-cluster
+  tb-core1:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-core2:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-rule-engine1:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-rule-engine2:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-mqtt-transport1:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-mqtt-transport2:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-http-transport1:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-http-transport2:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-coap-transport:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-lwm2m-transport:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-snmp-transport:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-vc-executor1:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
+  tb-vc-executor2:
+    env_file:
+      - cache-redis-cluster.env
+    depends_on:
+      - redis-node-5
--- a/docker/docker-compose.redis.volumes.yml
+++ b/docker/docker-compose.redis.volumes.yml
@ -0,0 +1,27 @@
+#
+# Copyright © 2016-2022 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+services:
+  redis:
+    volumes:
+      - redis-data:/bitnami/redis/data
+
+volumes:
+  redis-data:
+    external: true
+    name: ${REDIS_DATA_VOLUME}
--- a/docker/docker-compose.redis.yml
+++ b/docker/docker-compose.redis.yml
@ -0,0 +1,97 @@
+#
+# Copyright © 2016-2022 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+services:
+# Redis standalone
+  redis:
+    restart: always
+    image: bitnami/redis:7.0
+    environment:
+      # ALLOW_EMPTY_PASSWORD is recommended only for development.
+      ALLOW_EMPTY_PASSWORD: "yes"
+    ports:
+      - '6379:6379'
+    volumes:
+      - ./tb-node/redis-data:/bitnami/redis/data
+
+# ThingsBoard setup to use redis-standalone
+  tb-core1:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-core2:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-rule-engine1:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-rule-engine2:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-mqtt-transport1:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-mqtt-transport2:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-http-transport1:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-http-transport2:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-coap-transport:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-lwm2m-transport:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-snmp-transport:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-vc-executor1:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
+  tb-vc-executor2:
+    env_file:
+      - cache-redis.env
+    depends_on:
+      - redis
--- a/docker/docker-compose.volumes.yml
+++ b/docker/docker-compose.volumes.yml
@ -0,0 +1,81 @@
+#
+# Copyright © 2016-2022 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+services:
+  tb-core1:
+    volumes:
+      - tb-log-volume:/var/log/thingsboard
+  tb-core2:
+    volumes:
+      - tb-log-volume:/var/log/thingsboard
+  tb-rule-engine1:
+    volumes:
+      - tb-log-volume:/var/log/thingsboard
+  tb-rule-engine2:
+    volumes:
+      - tb-log-volume:/var/log/thingsboard
+  tb-coap-transport:
+    volumes:
+      - tb-coap-transport-log-volume:/var/log/tb-coap-transport
+  tb-lwm2m-transport:
+    volumes:
+      - tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport
+  tb-http-transport1:
+    volumes:
+      - tb-http-transport-log-volume:/var/log/tb-http-transport
+  tb-http-transport2:
+    volumes:
+      - tb-http-transport-log-volume:/var/log/tb-http-transport
+  tb-mqtt-transport1:
+    volumes:
+      - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
+  tb-mqtt-transport2:
+    volumes:
+      - tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
+  tb-snmp-transport:
+    volumes:
+      - tb-snmp-transport-log-volume:/var/log/tb-snmp-transport
+  tb-vc-executor1:
+    volumes:
+      - tb-vc-executor-log-volume:/var/log/tb-vc-executor
+  tb-vc-executor2:
+    volumes:
+      - tb-vc-executor-log-volume:/var/log/tb-vc-executor
+
+volumes:
+  tb-log-volume:
+    external: true
+    name: ${TB_LOG_VOLUME}
+  tb-coap-transport-log-volume:
+    external: true
+    name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
+  tb-lwm2m-transport-log-volume:
+    external: true
+    name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME}
+  tb-http-transport-log-volume:
+    external: true
+    name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
+  tb-mqtt-transport-log-volume:
+    external: true
+    name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}
+  tb-snmp-transport-log-volume:
+    external: true
+    name: ${TB_SNMP_TRANSPORT_LOG_VOLUME}
+  tb-vc-executor-log-volume:
+    external: true
+    name: ${TB_VC_EXECUTOR_LOG_VOLUME}
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@ -26,15 +26,10 @@ services:
    environment:
      ZOO_MY_ID: 1
      ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181
-  redis:
-    restart: always
-    image: redis:4.0
-    ports:
-      - "6379"
  tb-js-executor:
    restart: always
    image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}"
-    scale: 20
+    scale: 10
    env_file:
      - tb-js-executor.env
  tb-core1:
@ -59,7 +54,6 @@ services:
      - ./tb-node/log:/var/log/thingsboard
    depends_on:
      - zookeeper
-      - redis
      - tb-js-executor
      - tb-rule-engine1
      - tb-rule-engine2
@ -85,7 +79,6 @@ services:
      - ./tb-node/log:/var/log/thingsboard
    depends_on:
      - zookeeper
-      - redis
      - tb-js-executor
      - tb-rule-engine1
      - tb-rule-engine2
@ -109,7 +102,6 @@ services:
      - ./tb-node/log:/var/log/thingsboard
    depends_on:
      - zookeeper
-      - redis
      - tb-js-executor
  tb-rule-engine2:
    restart: always
@ -131,7 +123,6 @@ services:
      - ./tb-node/log:/var/log/thingsboard
    depends_on:
      - zookeeper
-      - redis
      - tb-js-executor
  tb-mqtt-transport1:
    restart: always
--- a/docker/docker-create-log-folders.sh
+++ b/docker/docker-create-log-folders.sh
@ -28,3 +28,24 @@ mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log
 mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log

 mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log
+
+mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/
+
+source .env
+CACHE="${CACHE:-redis}"
+case $CACHE in
+    redis)
+    mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/
+    ;;
+    redis-cluster)
+    mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/
+    mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/
+    mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/
+    mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/
+    mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/
+    mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/
+    ;;
+    *)
+    echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
+    exit 1
+esac
--- a/docker/docker-install-tb.sh
+++ b/docker/docker-install-tb.sh
@ -45,12 +45,14 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?

 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?

+ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
+
 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?

 if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
-    docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
+    docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
 fi

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1


--- a/docker/docker-remove-services.sh
+++ b/docker/docker-remove-services.sh
@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?

 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?

+ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
+
 ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v
--- a/docker/docker-start-services.sh
+++ b/docker/docker-start-services.sh
@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?

 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?

+ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
+
 ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d
--- a/docker/docker-stop-services.sh
+++ b/docker/docker-stop-services.sh
@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?

 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?

+ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
+
 ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop
--- a/docker/docker-update-service.sh
+++ b/docker/docker-update-service.sh
@ -23,5 +23,7 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?

 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@
+ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
+
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@
--- a/docker/docker-upgrade-tb.sh
+++ b/docker/docker-upgrade-tb.sh
@ -44,10 +44,12 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?

 ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?

+ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
+
 ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES

-docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1
+docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1
--- a/docker/tb-coap-transport.env
+++ b/docker/tb-coap-transport.env
@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
 WEB_APPLICATION_ENABLE=true
 WEB_APPLICATION_TYPE=servlet
 HTTP_BIND_PORT=8081
-
-CACHE_TYPE=redis
-REDIS_HOST=redis
--- a/docker/tb-http-transport.env
+++ b/docker/tb-http-transport.env
@ -7,6 +7,3 @@ HTTP_REQUEST_TIMEOUT=60000

 METRICS_ENABLED=true
 METRICS_ENDPOINTS_EXPOSE=prometheus
-
-CACHE_TYPE=redis
-REDIS_HOST=redis
--- a/docker/tb-lwm2m-transport.env
+++ b/docker/tb-lwm2m-transport.env
@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
 WEB_APPLICATION_ENABLE=true
 WEB_APPLICATION_TYPE=servlet
 HTTP_BIND_PORT=8081
-
-CACHE_TYPE=redis
-REDIS_HOST=redis
--- a/docker/tb-mqtt-transport.env
+++ b/docker/tb-mqtt-transport.env
@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
 WEB_APPLICATION_ENABLE=true
 WEB_APPLICATION_TYPE=servlet
 HTTP_BIND_PORT=8081
-
-CACHE_TYPE=redis
-REDIS_HOST=redis
--- a/docker/tb-node.env
+++ b/docker/tb-node.env
@ -4,8 +4,6 @@ ZOOKEEPER_ENABLED=true
 ZOOKEEPER_URL=zookeeper:2181
 JS_EVALUATOR=remote
 TRANSPORT_TYPE=remote
-CACHE_TYPE=redis
-REDIS_HOST=redis

 HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false

--- a/docker/tb-snmp-transport.env
+++ b/docker/tb-snmp-transport.env
@ -6,6 +6,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
 WEB_APPLICATION_ENABLE=true
 WEB_APPLICATION_TYPE=servlet
 HTTP_BIND_PORT=8081
-
-CACHE_TYPE=redis
-REDIS_HOST=redis
--- a/msa/black-box-tests/README.md
+++ b/msa/black-box-tests/README.md
@ -18,8 +18,13 @@ As result, in REPOSITORY column, next images should be present:
        thingsboard/tb-web-ui
        thingsboard/tb-js-executor

- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory:
+- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone:

        mvn clean install -DblackBoxTests.skip=false

+- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis cluster:
+
+        mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true
+
+

--- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java
+++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java
@ -39,7 +39,7 @@ import static org.junit.Assert.fail;
@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"})
@Slf4j
 public class ContainerTestSuite {
-
+    final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster"));
    private static final String SOURCE_DIR = "./../../docker/";
    private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*";
    private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*";
@ -52,6 +52,7 @@ public class ContainerTestSuite {
    @ClassRule
    public static DockerComposeContainer getTestContainer() {
        if (testContainer == null) {
+            log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER);
            boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers"));
            try {
                final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/";
@ -73,9 +74,17 @@ public class ContainerTestSuite {

                testContainer = new DockerComposeContainerImpl<>(
                        new File(targetDir + "docker-compose.yml"),
+                        new File(targetDir + "docker-compose.volumes.yml"),
                        new File(targetDir + "docker-compose.postgres.yml"),
                        new File(targetDir + "docker-compose.postgres.volumes.yml"),
-                        new File(targetDir + "docker-compose.kafka.yml"))
+                        new File(targetDir + "docker-compose.kafka.yml"),
+                        IS_REDIS_CLUSTER
+                                ? new File("./../../docker/docker-compose.redis-cluster.yml")
+                                : new File("./../../docker/docker-compose.redis.yml"),
+                        IS_REDIS_CLUSTER
+                                ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml")
+                                : new File("./../../docker/docker-compose.redis.volumes.yml")
+                        )
                        .withPull(false)
                        .withLocalCompose(true)
                        .withTailChildContainers(!skipTailChildContainers)
--- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java
+++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java
@ -15,6 +15,7 @@
 */
 package org.thingsboard.server.msa;

+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.RandomStringUtils;
 import org.junit.rules.ExternalResource;
 import org.testcontainers.utility.Base58;
@ -24,10 +25,16 @@ import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;

+@Slf4j
 public class ThingsBoardDbInstaller extends ExternalResource {

+    final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster"));
    private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume";
+    private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume";
+    private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume";
    private final static String TB_LOG_VOLUME = "tb-log-test-volume";
    private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume";
    private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume";
@ -39,6 +46,9 @@ public class ThingsBoardDbInstaller extends ExternalResource {
    private final DockerComposeExecutor dockerCompose;

    private final String postgresDataVolume;
+
+    private final String redisDataVolume;
+    private final String redisClusterDataVolume;
    private final String tbLogVolume;
    private final String tbCoapTransportLogVolume;
    private final String tbLwm2mTransportLogVolume;
@ -49,14 +59,26 @@ public class ThingsBoardDbInstaller extends ExternalResource {
    private final Map<String, String> env;

    public ThingsBoardDbInstaller() {
-        List<File> composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"),
+        log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER);
+        List<File> composeFiles = Arrays.asList(
+                new File("./../../docker/docker-compose.yml"),
+                new File("./../../docker/docker-compose.volumes.yml"),
                new File("./../../docker/docker-compose.postgres.yml"),
-                new File("./../../docker/docker-compose.postgres.volumes.yml"));
+                new File("./../../docker/docker-compose.postgres.volumes.yml"),
+                IS_REDIS_CLUSTER
+                        ? new File("./../../docker/docker-compose.redis-cluster.yml")
+                        : new File("./../../docker/docker-compose.redis.yml"),
+                IS_REDIS_CLUSTER
+                        ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml")
+                        : new File("./../../docker/docker-compose.redis.volumes.yml")
+        );

        String identifier = Base58.randomString(6).toLowerCase();
        String project = identifier + Base58.randomString(6).toLowerCase();

        postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME;
+        redisDataVolume = project + "_" + REDIS_DATA_VOLUME;
+        redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME;
        tbLogVolume = project + "_" + TB_LOG_VOLUME;
        tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME;
        tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME;
@ -76,6 +98,13 @@ public class ThingsBoardDbInstaller extends ExternalResource {
        env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume);
        env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume);
        env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume);
+        if (IS_REDIS_CLUSTER) {
+            for (int i = 0; i < 6; i++) {
+                env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i);
+            }
+        } else {
+            env.put("REDIS_DATA_VOLUME", redisDataVolume);
+        }
        dockerCompose.withEnv(env);
    }

@ -111,7 +140,20 @@ public class ThingsBoardDbInstaller extends ExternalResource {
            dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume);
            dockerCompose.invokeDocker();

-            dockerCompose.withCommand("up -d redis postgres");
+            String redisService = "";
+            if (IS_REDIS_CLUSTER) {
+                for (int i = 0; i < 6; i++) {
+                    redisService = redisService + " redis-node-" + i;
+                    dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i);
+                    dockerCompose.invokeDocker();
+                }
+            } else {
+                redisService = "redis";
+                dockerCompose.withCommand("volume create " + redisDataVolume);
+                dockerCompose.invokeDocker();
+            }
+
+            dockerCompose.withCommand("up -d postgres " + redisService);
            dockerCompose.invokeCompose();

            dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1");
@ -137,7 +179,10 @@ public class ThingsBoardDbInstaller extends ExternalResource {

        dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume +
                " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume +
-                " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume);
+                " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume +
+                (IS_REDIS_CLUSTER
+                        ? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining())
+                        : redisDataVolume));
        dockerCompose.invokeDocker();
    }