diff --git a/docker/.env b/docker/.env index c03845dfe0..0138501ebc 100644 --- a/docker/.env +++ b/docker/.env @@ -15,4 +15,6 @@ TB_VERSION=latest DATABASE=postgres +LOAD_BALANCER_NAME=haproxy-certbot + KAFKA_TOPICS="js.eval.requests:100:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.transport.api.requests:30:1:delete --config=retention.ms=60000 --config=segment.bytes=26214400 --config=retention.bytes=104857600,tb.rule-engine:30:1" diff --git a/docker/docker-compose.postgres.volumes.yml b/docker/docker-compose.postgres.volumes.yml new file mode 100644 index 0000000000..d94b066cb0 --- /dev/null +++ b/docker/docker-compose.postgres.volumes.yml @@ -0,0 +1,36 @@ +# +# Copyright © 2016-2018 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + postgres: + volumes: + - postgres-db-volume:/var/lib/postgresql/data + tb1: + volumes: + - tb-log-volume:/var/log/thingsboard + tb2: + volumes: + - tb-log-volume:/var/log/thingsboard + +volumes: + postgres-db-volume: + external: true + name: ${POSTGRES_DATA_VOLUME} + tb-log-volume: + external: true + name: ${TB_LOG_VOLUME} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1741db3694..0a09d8a024 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -145,7 +145,7 @@ services: - tb-web-ui.env haproxy: restart: always - container_name: haproxy-certbot + container_name: "${LOAD_BALANCER_NAME}" image: xalauc/haproxy-certbot:1.7.9 volumes: - ./haproxy/config:/config diff --git a/docker/tb-node/conf/logback.xml b/docker/tb-node/conf/logback.xml index 1c69f537bc..6ec2d0b2d0 100644 --- a/docker/tb-node/conf/logback.xml +++ b/docker/tb-node/conf/logback.xml @@ -24,7 +24,7 @@ /var/log/thingsboard/${TB_HOST}/thingsboard.log - /var/log/thingsboard/thingsboard.%d{yyyy-MM-dd}.%i.log + /var/log/thingsboard/${TB_HOST}/thingsboard.%d{yyyy-MM-dd}.%i.log 100MB 30 3GB diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 2af4d2c14e..9f5ac0912c 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -36,6 +36,7 @@ ${basedir}/../.. true 1.9.1 + 1.10 1.3.9 4.5.6 @@ -46,6 +47,11 @@ testcontainers ${testcontainers.version} + + org.zeroturnaround + zt-exec + ${zeroturnaround.version} + org.java-websocket Java-WebSocket diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 495fd94d2e..3233617823 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -17,23 +17,43 @@ package org.thingsboard.server.msa; import org.junit.ClassRule; import org.junit.extensions.cpsuite.ClasspathSuite; +import org.junit.rules.ExternalResource; import org.junit.runner.RunWith; import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.Base58; import java.io.File; import java.time.Duration; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @RunWith(ClasspathSuite.class) @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) public class ContainerTestSuite { + private static DockerComposeContainer testContainer; + + @ClassRule + public static ThingsBoardDbInstaller installTb = new ThingsBoardDbInstaller(); + @ClassRule - public static DockerComposeContainer composeContainer = new DockerComposeContainer( - new File("./../../docker/docker-compose.yml"), - new File("./../../docker/docker-compose.postgres.yml")) - .withPull(false) - .withLocalCompose(true) - .withTailChildContainers(true) - .withExposedService("tb-web-ui1", 8080, Wait.forHttp("/login").withStartupTimeout(Duration.ofSeconds(120))); + public static DockerComposeContainer getTestContainer() { + if (testContainer == null) { + testContainer = new DockerComposeContainer( + new File("./../../docker/docker-compose.yml"), + new File("./../../docker/docker-compose.postgres.yml"), + new File("./../../docker/docker-compose.postgres.volumes.yml")) + .withPull(false) + .withLocalCompose(true) + .withTailChildContainers(true) + .withEnv("POSTGRES_DATA_VOLUME", installTb.getPostgresDataVolume()) + .withEnv("TB_LOG_VOLUME", installTb.getTbLogVolume()) + .withEnv("LOAD_BALANCER_NAME", "") + .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(Duration.ofSeconds(120))); + } + return testContainer; + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/DockerComposeExecutor.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/DockerComposeExecutor.java new file mode 100644 index 0000000000..25d2e6ee4c --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/DockerComposeExecutor.java @@ -0,0 +1,119 @@ +/** + * Copyright © 2016-2018 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.thingsboard.server.msa; + + +import com.google.common.base.Splitter; +import com.google.common.collect.Maps; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.SystemUtils; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.utility.CommandLine; +import org.zeroturnaround.exec.InvalidExitValueException; +import org.zeroturnaround.exec.ProcessExecutor; +import org.zeroturnaround.exec.stream.slf4j.Slf4jStream; + +import java.io.File; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Stream; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.stream.Collectors.joining; + +@Slf4j +public class DockerComposeExecutor { + + String ENV_PROJECT_NAME = "COMPOSE_PROJECT_NAME"; + String ENV_COMPOSE_FILE = "COMPOSE_FILE"; + + private static final String COMPOSE_EXECUTABLE = SystemUtils.IS_OS_WINDOWS ? "docker-compose.exe" : "docker-compose"; + private static final String DOCKER_EXECUTABLE = SystemUtils.IS_OS_WINDOWS ? "docker.exe" : "docker"; + + private final List composeFiles; + private final String identifier; + private String cmd = ""; + private Map env = new HashMap<>(); + + public DockerComposeExecutor(List composeFiles, String identifier) { + validateFileList(composeFiles); + this.composeFiles = composeFiles; + this.identifier = identifier; + } + + public DockerComposeExecutor withCommand(String cmd) { + this.cmd = cmd; + return this; + } + + public DockerComposeExecutor withEnv(Map env) { + this.env = env; + return this; + } + + public void invokeCompose() { + // bail out early + if (!CommandLine.executableExists(COMPOSE_EXECUTABLE)) { + throw new ContainerLaunchException("Local Docker Compose not found. Is " + COMPOSE_EXECUTABLE + " on the PATH?"); + } + final Map environment = Maps.newHashMap(env); + environment.put(ENV_PROJECT_NAME, identifier); + final Stream absoluteDockerComposeFilePaths = composeFiles.stream().map(File::getAbsolutePath).map(Objects::toString); + final String composeFileEnvVariableValue = absoluteDockerComposeFilePaths.collect(joining(File.pathSeparator + "")); + log.debug("Set env COMPOSE_FILE={}", composeFileEnvVariableValue); + final File pwd = composeFiles.get(0).getAbsoluteFile().getParentFile().getAbsoluteFile(); + environment.put(ENV_COMPOSE_FILE, composeFileEnvVariableValue); + log.info("Local Docker Compose is running command: {}", cmd); + final List command = Splitter.onPattern(" ").omitEmptyStrings().splitToList(COMPOSE_EXECUTABLE + " " + cmd); + try { + new ProcessExecutor().command(command).redirectOutput(Slf4jStream.of(log).asInfo()).redirectError(Slf4jStream.of(log).asError()).environment(environment).directory(pwd).exitValueNormal().executeNoTimeout(); + log.info("Docker Compose has finished running"); + } catch (InvalidExitValueException e) { + throw new ContainerLaunchException("Local Docker Compose exited abnormally with code " + e.getExitValue() + " whilst running command: " + cmd); + } catch (Exception e) { + throw new ContainerLaunchException("Error running local Docker Compose command: " + cmd, e); + } + } + + public void invokeDocker() { + // bail out early + if (!CommandLine.executableExists(DOCKER_EXECUTABLE)) { + throw new ContainerLaunchException("Local Docker not found. Is " + DOCKER_EXECUTABLE + " on the PATH?"); + } + final File pwd = composeFiles.get(0).getAbsoluteFile().getParentFile().getAbsoluteFile(); + log.info("Local Docker is running command: {}", cmd); + final List command = Splitter.onPattern(" ").omitEmptyStrings().splitToList(DOCKER_EXECUTABLE + " " + cmd); + try { + new ProcessExecutor().command(command).redirectOutput(Slf4jStream.of(log).asInfo()).redirectError(Slf4jStream.of(log).asError()).directory(pwd).exitValueNormal().executeNoTimeout(); + log.info("Docker has finished running"); + } catch (InvalidExitValueException e) { + throw new ContainerLaunchException("Local Docker exited abnormally with code " + e.getExitValue() + " whilst running command: " + cmd); + } catch (Exception e) { + throw new ContainerLaunchException("Error running local Docker command: " + cmd, e); + } + } + + void validateFileList(List composeFiles) { + checkNotNull(composeFiles); + checkArgument(!composeFiles.isEmpty(), "No docker compose file have been provided"); + } + + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java new file mode 100644 index 0000000000..0a902b9c5e --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -0,0 +1,106 @@ +/** + * Copyright © 2016-2018 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import org.junit.rules.ExternalResource; +import org.testcontainers.utility.Base58; + +import java.io.File; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ThingsBoardDbInstaller extends ExternalResource { + + private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; + private final static String TB_LOG_VOLUME = "tb-log-test-volume"; + + private final DockerComposeExecutor dockerCompose; + + private final String postgresDataVolume; + private final String tbLogVolume; + + public ThingsBoardDbInstaller() { + List composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), + new File("./../../docker/docker-compose.postgres.yml"), + new File("./../../docker/docker-compose.postgres.volumes.yml")); + + String identifier = Base58.randomString(6).toLowerCase(); + String project = identifier + Base58.randomString(6).toLowerCase(); + + postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; + tbLogVolume = project + "_" + TB_LOG_VOLUME; + + dockerCompose = new DockerComposeExecutor(composeFiles, project); + + Map env = new HashMap<>(); + env.put("POSTGRES_DATA_VOLUME", postgresDataVolume); + env.put("TB_LOG_VOLUME", tbLogVolume); + dockerCompose.withEnv(env); + } + + public String getPostgresDataVolume() { + return postgresDataVolume; + } + + public String getTbLogVolume() { + return tbLogVolume; + } + + @Override + protected void before() throws Throwable { + try { + + dockerCompose.withCommand("volume create " + postgresDataVolume); + dockerCompose.invokeDocker(); + + dockerCompose.withCommand("volume create " + tbLogVolume); + dockerCompose.invokeDocker(); + + dockerCompose.withCommand("up -d redis postgres"); + dockerCompose.invokeCompose(); + + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb1"); + dockerCompose.invokeCompose(); + + } finally { + try { + dockerCompose.withCommand("down -v"); + dockerCompose.invokeCompose(); + } catch (Exception e) {} + } + } + + @Override + protected void after() { + File tbLogsDir = new File("./target/tb-logs/"); + tbLogsDir.mkdirs(); + + dockerCompose.withCommand("run -d --rm --name tb-logs-container -v " + tbLogVolume + ":/root alpine tail -f /dev/null"); + dockerCompose.invokeDocker(); + + dockerCompose.withCommand("cp tb-logs-container:/root/. "+tbLogsDir.getAbsolutePath()); + dockerCompose.invokeDocker(); + + dockerCompose.withCommand("rm -f tb-logs-container"); + dockerCompose.invokeDocker(); + + dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume); + dockerCompose.invokeDocker(); + } + +}