Update black box tests infrastructure.

docker/.env

@@ -15,4 +15,6 @@ TB_VERSION=latest
 
 DATABASE=postgres
 
+LOAD_BALANCER_NAME=haproxy-certbot
+
 KAFKA_TOPICS="js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1"

docker/docker-compose.postgres.volumes.yml

@@ -0,0 +1,36 @@
+#
+# Copyright © 2016-2018 The Thingsboard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: '2.2'
+
+services:
+  postgres:
+    volumes:
+      - postgres-db-volume:/var/lib/postgresql/data
+  tb1:
+    volumes:
+      - tb-log-volume:/var/log/thingsboard
+  tb2:
+    volumes:
+      - tb-log-volume:/var/log/thingsboard
+
+volumes:
+  postgres-db-volume:
+    external: true
+    name: ${POSTGRES_DATA_VOLUME}
+  tb-log-volume:
+    external: true
+    name: ${TB_LOG_VOLUME}

docker/docker-compose.yml

@@ -145,7 +145,7 @@ services:
       - tb-web-ui.env
   haproxy:
     restart: always
-    container_name: haproxy-certbot
+    container_name: "${LOAD_BALANCER_NAME}"
     image: xalauc/haproxy-certbot:1.7.9
     volumes:
      - ./haproxy/config:/config

docker/tb-node/conf/logback.xml

@@ -24,7 +24,7 @@
         <file>/var/log/thingsboard/${TB_HOST}/thingsboard.log</file>
         <rollingPolicy
                 class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>/var/log/thingsboard/thingsboard.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>/var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
             <maxFileSize>100MB</maxFileSize>
             <maxHistory>30</maxHistory>
             <totalSizeCap>3GB</totalSizeCap>

msa/black-box-tests/pom.xml

@@ -36,6 +36,7 @@
         <main.dir>${basedir}/../..</main.dir>
         <blackBoxTests.skip>true</blackBoxTests.skip>
         <testcontainers.version>1.9.1</testcontainers.version>
+        <zeroturnaround.version>1.10</zeroturnaround.version>
         <java-websocket.version>1.3.9</java-websocket.version>
         <httpclient.version>4.5.6</httpclient.version>
     </properties>
@@ -46,6 +47,11 @@
             <artifactId>testcontainers</artifactId>
             <version>${testcontainers.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.zeroturnaround</groupId>
+            <artifactId>zt-exec</artifactId>
+            <version>${zeroturnaround.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.java-websocket</groupId>
             <artifactId>Java-WebSocket</artifactId>

msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@@ -17,23 +17,43 @@ package org.thingsboard.server.msa;
 
 import org.junit.ClassRule;
 import org.junit.extensions.cpsuite.ClasspathSuite;
+import org.junit.rules.ExternalResource;
 import org.junit.runner.RunWith;
 import org.testcontainers.containers.DockerComposeContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
+import org.testcontainers.utility.Base58;
 
 import java.io.File;
 import java.time.Duration;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 @RunWith(ClasspathSuite.class)
 @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"})
 public class ContainerTestSuite {
 
+    private static DockerComposeContainer testContainer;
+
+    @ClassRule
+    public static ThingsBoardDbInstaller installTb = new ThingsBoardDbInstaller();
+
     @ClassRule
-    public static DockerComposeContainer composeContainer = new DockerComposeContainer(
+    public static DockerComposeContainer getTestContainer() {
-            new File("./../../docker/docker-compose.yml"),
+        if (testContainer == null) {
-            new File("./../../docker/docker-compose.postgres.yml"))
+            testContainer = new DockerComposeContainer(
-            .withPull(false)
+                    new File("./../../docker/docker-compose.yml"),
-            .withLocalCompose(true)
+                    new File("./../../docker/docker-compose.postgres.yml"),
-            .withTailChildContainers(true)
+                    new File("./../../docker/docker-compose.postgres.volumes.yml"))
-            .withExposedService("tb-web-ui1", 8080, Wait.forHttp("/login").withStartupTimeout(Duration.ofSeconds(120)));
+                    .withPull(false)
+                    .withLocalCompose(true)
+                    .withTailChildContainers(true)
+                    .withEnv("POSTGRES_DATA_VOLUME", installTb.getPostgresDataVolume())
+                    .withEnv("TB_LOG_VOLUME", installTb.getTbLogVolume())
+                    .withEnv("LOAD_BALANCER_NAME", "")
+                    .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(Duration.ofSeconds(120)));
+        }
+        return testContainer;
+    }
 }

msa/black-box-tests/src/test/java/org/thingsboard/server/msa/DockerComposeExecutor.java

@@ -0,0 +1,119 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.thingsboard.server.msa;
+
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Maps;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.SystemUtils;
+import org.testcontainers.containers.ContainerLaunchException;
+import org.testcontainers.utility.CommandLine;
+import org.zeroturnaround.exec.InvalidExitValueException;
+import org.zeroturnaround.exec.ProcessExecutor;
+import org.zeroturnaround.exec.stream.slf4j.Slf4jStream;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.stream.Collectors.joining;
+
+@Slf4j
+public class DockerComposeExecutor {
+
+    String ENV_PROJECT_NAME = "COMPOSE_PROJECT_NAME";
+    String ENV_COMPOSE_FILE = "COMPOSE_FILE";
+
+    private static final String COMPOSE_EXECUTABLE = SystemUtils.IS_OS_WINDOWS ? "docker-compose.exe" : "docker-compose";
+    private static final String DOCKER_EXECUTABLE = SystemUtils.IS_OS_WINDOWS ? "docker.exe" : "docker";
+
+    private final List<File> composeFiles;
+    private final String identifier;
+    private String cmd = "";
+    private Map<String, String> env = new HashMap<>();
+
+    public DockerComposeExecutor(List<File> composeFiles, String identifier) {
+        validateFileList(composeFiles);
+        this.composeFiles = composeFiles;
+        this.identifier = identifier;
+    }
+
+    public DockerComposeExecutor withCommand(String cmd) {
+        this.cmd = cmd;
+        return this;
+    }
+
+    public DockerComposeExecutor withEnv(Map<String, String> env) {
+        this.env = env;
+        return this;
+    }
+
+    public void invokeCompose() {
+        // bail out early
+        if (!CommandLine.executableExists(COMPOSE_EXECUTABLE)) {
+            throw new ContainerLaunchException("Local Docker Compose not found. Is " + COMPOSE_EXECUTABLE + " on the PATH?");
+        }
+        final Map<String, String> environment = Maps.newHashMap(env);
+        environment.put(ENV_PROJECT_NAME, identifier);
+        final Stream<String> absoluteDockerComposeFilePaths = composeFiles.stream().map(File::getAbsolutePath).map(Objects::toString);
+        final String composeFileEnvVariableValue = absoluteDockerComposeFilePaths.collect(joining(File.pathSeparator + ""));
+        log.debug("Set env COMPOSE_FILE={}", composeFileEnvVariableValue);
+        final File pwd = composeFiles.get(0).getAbsoluteFile().getParentFile().getAbsoluteFile();
+        environment.put(ENV_COMPOSE_FILE, composeFileEnvVariableValue);
+        log.info("Local Docker Compose is running command: {}", cmd);
+        final List<String> command = Splitter.onPattern(" ").omitEmptyStrings().splitToList(COMPOSE_EXECUTABLE + " " + cmd);
+        try {
+            new ProcessExecutor().command(command).redirectOutput(Slf4jStream.of(log).asInfo()).redirectError(Slf4jStream.of(log).asError()).environment(environment).directory(pwd).exitValueNormal().executeNoTimeout();
+            log.info("Docker Compose has finished running");
+        } catch (InvalidExitValueException e) {
+            throw new ContainerLaunchException("Local Docker Compose exited abnormally with code " + e.getExitValue() + " whilst running command: " + cmd);
+        } catch (Exception e) {
+            throw new ContainerLaunchException("Error running local Docker Compose command: " + cmd, e);
+        }
+    }
+
+    public void invokeDocker() {
+        // bail out early
+        if (!CommandLine.executableExists(DOCKER_EXECUTABLE)) {
+            throw new ContainerLaunchException("Local Docker not found. Is " + DOCKER_EXECUTABLE + " on the PATH?");
+        }
+        final File pwd = composeFiles.get(0).getAbsoluteFile().getParentFile().getAbsoluteFile();
+        log.info("Local Docker is running command: {}", cmd);
+        final List<String> command = Splitter.onPattern(" ").omitEmptyStrings().splitToList(DOCKER_EXECUTABLE + " " + cmd);
+        try {
+            new ProcessExecutor().command(command).redirectOutput(Slf4jStream.of(log).asInfo()).redirectError(Slf4jStream.of(log).asError()).directory(pwd).exitValueNormal().executeNoTimeout();
+            log.info("Docker has finished running");
+        } catch (InvalidExitValueException e) {
+            throw new ContainerLaunchException("Local Docker exited abnormally with code " + e.getExitValue() + " whilst running command: " + cmd);
+        } catch (Exception e) {
+            throw new ContainerLaunchException("Error running local Docker command: " + cmd, e);
+        }
+    }
+
+    void validateFileList(List<File> composeFiles) {
+        checkNotNull(composeFiles);
+        checkArgument(!composeFiles.isEmpty(), "No docker compose file have been provided");
+    }
+
+
+}

msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java

@@ -0,0 +1,106 @@
+/**
+ * Copyright © 2016-2018 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.msa;
+
+import org.junit.rules.ExternalResource;
+import org.testcontainers.utility.Base58;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ThingsBoardDbInstaller extends ExternalResource {
+
+    private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume";
+    private final static String TB_LOG_VOLUME = "tb-log-test-volume";
+
+    private final DockerComposeExecutor dockerCompose;
+
+    private final String postgresDataVolume;
+    private final String tbLogVolume;
+
+    public ThingsBoardDbInstaller() {
+        List<File> composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"),
+                new File("./../../docker/docker-compose.postgres.yml"),
+                new File("./../../docker/docker-compose.postgres.volumes.yml"));
+
+        String identifier = Base58.randomString(6).toLowerCase();
+        String project = identifier + Base58.randomString(6).toLowerCase();
+
+        postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME;
+        tbLogVolume = project + "_" + TB_LOG_VOLUME;
+
+        dockerCompose = new DockerComposeExecutor(composeFiles, project);
+
+        Map<String, String> env = new HashMap<>();
+        env.put("POSTGRES_DATA_VOLUME", postgresDataVolume);
+        env.put("TB_LOG_VOLUME", tbLogVolume);
+        dockerCompose.withEnv(env);
+    }
+
+    public String getPostgresDataVolume() {
+        return postgresDataVolume;
+    }
+
+    public String getTbLogVolume() {
+        return tbLogVolume;
+    }
+
+    @Override
+    protected void before() throws Throwable {
+        try {
+
+            dockerCompose.withCommand("volume create " + postgresDataVolume);
+            dockerCompose.invokeDocker();
+
+            dockerCompose.withCommand("volume create " + tbLogVolume);
+            dockerCompose.invokeDocker();
+
+            dockerCompose.withCommand("up -d redis postgres");
+            dockerCompose.invokeCompose();
+
+            dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb1");
+            dockerCompose.invokeCompose();
+
+        } finally {
+            try {
+                dockerCompose.withCommand("down -v");
+                dockerCompose.invokeCompose();
+            } catch (Exception e) {}
+        }
+    }
+
+    @Override
+    protected void after() {
+        File tbLogsDir = new File("./target/tb-logs/");
+        tbLogsDir.mkdirs();
+
+        dockerCompose.withCommand("run -d --rm --name tb-logs-container -v " + tbLogVolume + ":/root alpine tail -f /dev/null");
+        dockerCompose.invokeDocker();
+
+        dockerCompose.withCommand("cp tb-logs-container:/root/. "+tbLogsDir.getAbsolutePath());
+        dockerCompose.invokeDocker();
+
+        dockerCompose.withCommand("rm -f tb-logs-container");
+        dockerCompose.invokeDocker();
+
+        dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume);
+        dockerCompose.invokeDocker();
+    }
+
+}