From 8a9ce8d27216dcabc06b86f12fecfbd98e91e8b7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 15:56:47 +0300 Subject: [PATCH 1/9] docker-compose: cleanup zookeeper dependency from hybrid. The necessary dependency in the main docker-compose.yml along with zookeeper service --- docker/docker-compose.hybrid.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index e3ba32b779..197e1f1dcb 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -38,27 +38,23 @@ services: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-core2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-rule-engine1: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-rule-engine2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra From 36fae79179431cb6b609f784f44c07659499c880 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 23 Jun 2022 17:27:12 +0300 Subject: [PATCH 2/9] Fix cluster issues --- .../edge/rpc/sync/DefaultEdgeRequestsService.java | 2 ++ .../service/entitiy/AbstractTbEntityService.java | 15 +++++++++++++-- .../entitiy/asset/DefaultTbAssetService.java | 2 +- .../customer/DefaultTbCustomerService.java | 2 +- .../dashboard/DefaultTbDashboardService.java | 2 +- .../entitiy/device/DefaultTbDeviceService.java | 2 +- .../DefaultTbDeviceProfileService.java | 2 +- .../install/DefaultSystemDataLoaderService.java | 2 ++ .../install/SqlDatabaseUpgradeService.java | 2 ++ .../install/update/DefaultDataUpdateService.java | 2 ++ .../dao/device/DeviceProfileServiceImpl.java | 2 ++ .../server/dao/queue/BaseQueueService.java | 2 ++ .../validator/DeviceProfileDataValidator.java | 2 +- .../dao/sqlts/BaseAbstractSqlTimeseriesDao.java | 3 ++- .../server/dao/tenant/TenantServiceImpl.java | 1 + 15 files changed, 34 insertions(+), 9 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index 90dbaf3312..210ad30c44 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -25,6 +25,7 @@ import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.Device; @@ -101,6 +102,7 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { @Autowired private DeviceService deviceService; + @Lazy @Autowired private TbEntityViewService entityViewService; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java index 7abe031f2e..e3d91b7a7e 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java @@ -65,6 +65,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.edge.EdgeNotificationService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; import org.thingsboard.server.service.install.InstallScripts; import org.thingsboard.server.service.ota.OtaPackageStateService; @@ -78,6 +79,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; import java.util.stream.Collectors; @Slf4j @@ -128,8 +130,8 @@ public abstract class AbstractTbEntityService { @Autowired protected DashboardService dashboardService; - @Autowired - protected EntitiesVersionControlService vcService; + @Autowired(required = false) + private EntitiesVersionControlService vcService; @Autowired protected EntityViewService entityViewService; @Lazy @@ -249,4 +251,13 @@ public abstract class AbstractTbEntityService { } return result; } + + protected ListenableFuture autoCommit(SecurityUser user, EntityId entityId) throws Exception { + if (vcService != null) { + return vcService.autoCommit(user, entityId); + } else { + // We do not support auto-commit for rule engine + return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!")); + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java index e118346588..e71c41b2f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java @@ -47,7 +47,7 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb TenantId tenantId = asset.getTenantId(); try { Asset savedAsset = checkNotNull(assetService.saveAsset(asset)); - vcService.autoCommit(user, savedAsset.getId()); + autoCommit(user, savedAsset.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), savedAsset, savedAsset.getCustomerId(), actionType, user); return savedAsset; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java index 536e1be976..4d5a85d811 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java @@ -42,7 +42,7 @@ public class DefaultTbCustomerService extends AbstractTbEntityService implements TenantId tenantId = customer.getTenantId(); try { Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer)); - vcService.autoCommit(user, savedCustomer.getId()); + autoCommit(user, savedCustomer.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, null, actionType, user); return savedCustomer; } catch (Exception e) { diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java index a475ff77da..c7a04fbb78 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java @@ -48,7 +48,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement TenantId tenantId = dashboard.getTenantId(); try { Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard)); - vcService.autoCommit(user, savedDashboard.getId()); + autoCommit(user, savedDashboard.getId()); notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard, null, actionType, user); return savedDashboard; diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java index 5199887940..5c232e4712 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java @@ -54,7 +54,7 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T ActionType actionType = device.getId() == null ? ActionType.ADDED : ActionType.UPDATED; try { Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken)); - vcService.autoCommit(user, savedDevice.getId()); + autoCommit(user, savedDevice.getId()); notificationEntityService.notifyCreateOrUpdateDevice(tenantId, savedDevice.getId(), savedDevice.getCustomerId(), savedDevice, oldDevice, actionType, user); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java index 3a0c1f1a23..a92e5109f1 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java @@ -54,7 +54,7 @@ public class DefaultTbDeviceProfileService extends AbstractTbEntityService imple } } DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile)); - vcService.autoCommit(user, savedDeviceProfile.getId()); + autoCommit(user, savedDeviceProfile.getId()); tbClusterService.onDeviceProfileChange(savedDeviceProfile, null); tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedDeviceProfile.getId(), actionType.equals(ActionType.ADDED) ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 50fcb28387..14e49e056f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; @@ -162,6 +163,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Getter private boolean persistActivityToTelemetry; + @Lazy @Autowired private QueueService queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java index 306102c66c..e428d55d91 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java @@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntitySubtype; @@ -121,6 +122,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService @Autowired private ApiUsageStateService apiUsageStateService; + @Lazy @Autowired private QueueService queueService; diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index c6576097ee..9bfa86b3ba 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -117,6 +118,7 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private TenantProfileService tenantProfileService; + @Lazy @Autowired private QueueService queueService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java index 95f2f034fe..a6cfa33fb8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.Device; @@ -69,6 +70,7 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService deviceProfileValidator; + @Lazy @Autowired private QueueService queueService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java index a0254a4923..45b5b2cc7e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.id.QueueId; @@ -43,6 +44,7 @@ public class BaseQueueService extends AbstractEntityService implements QueueServ @Autowired private QueueDao queueDao; + @Lazy @Autowired private TbTenantProfileCache tenantProfileCache; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java index 429a328c53..b1fb9b150b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java @@ -90,8 +90,8 @@ public class DeviceProfileDataValidator extends DataValidator { private DeviceDao deviceDao; @Autowired private TenantService tenantService; - @Autowired @Lazy + @Autowired private QueueService queueService; @Autowired private OtaPackageService otaPackageService; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java index dd9d15b8b1..82e54168bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java @@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DataIntegrityViolationException; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary; import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey; @@ -59,7 +60,7 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary); tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId()); keyId = saved.getKeyId(); - } catch (ConstraintViolationException e) { + } catch (DataIntegrityViolationException | ConstraintViolationException e) { tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey)); TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!")); tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index 35a22f8281..a6fefd3ead 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -107,6 +107,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService tenantValidator; + @Lazy @Autowired private QueueService queueService; From 011caa20fd3954378c13d2b5c74a1a4150ffc1d0 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Thu, 23 Jun 2022 18:58:29 +0300 Subject: [PATCH 3/9] Add hybrid mode for black-box tests. Update cassandra version in docker compose. Fix NPE in AbstractBufferedRateExecutor - ignore profile rate limits for SYS_TENANT_ID. --- .../util/AbstractBufferedRateExecutor.java | 9 +++- docker/docker-compose.cassandra.volumes.yml | 27 ++++++++++++ docker/docker-compose.hybrid.yml | 2 +- msa/black-box-tests/README.md | 4 ++ .../server/msa/ContainerTestSuite.java | 23 ++++++++--- .../server/msa/ThingsBoardDbInstaller.java | 41 ++++++++++++++----- 6 files changed, 86 insertions(+), 20 deletions(-) create mode 100644 docker/docker-compose.cassandra.volumes.yml diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java index 2850349470..4c45239921 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java @@ -116,8 +116,13 @@ public abstract class AbstractBufferedRateExecutor> extends DockerComposeContainer { - public DockerComposeContainerImpl(File... composeFiles) { + public DockerComposeContainerImpl(List composeFiles) { super(composeFiles); } @@ -72,10 +77,11 @@ public class ContainerTestSuite { } } - testContainer = new DockerComposeContainerImpl<>( - new File(targetDir + "docker-compose.yml"), + List composeFiles = new ArrayList<>(Arrays.asList(new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), - new File(targetDir + "docker-compose.postgres.yml"), + IS_HYBRID_MODE + ? new File(targetDir + "docker-compose.hybrid.yml") + : new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER @@ -83,8 +89,13 @@ public class ContainerTestSuite { : new File("./../../docker/docker-compose.redis.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") - : new File("./../../docker/docker-compose.redis.volumes.yml") - ) + : new File("./../../docker/docker-compose.redis.volumes.yml"))); + + if (IS_HYBRID_MODE) { + composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); + } + + testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) .withLocalCompose(true) .withTailChildContainers(!skipTailChildContainers) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 08a333ab52..08cadbceb2 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -21,10 +21,7 @@ import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import java.io.File; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -32,7 +29,10 @@ import java.util.stream.IntStream; public class ThingsBoardDbInstaller extends ExternalResource { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); + final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; + + private final static String CASSANDRA_DATA_VOLUME = "tb-cassandra-test-data-volume"; private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume"; private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume"; @@ -46,6 +46,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { private final DockerComposeExecutor dockerCompose; private final String postgresDataVolume; + private final String cassandraDataVolume; private final String redisDataVolume; private final String redisClusterDataVolume; @@ -60,10 +61,13 @@ public class ThingsBoardDbInstaller extends ExternalResource { public ThingsBoardDbInstaller() { log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); - List composeFiles = Arrays.asList( + log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); + List composeFiles = new ArrayList<>(Arrays.asList( new File("./../../docker/docker-compose.yml"), new File("./../../docker/docker-compose.volumes.yml"), - new File("./../../docker/docker-compose.postgres.yml"), + IS_HYBRID_MODE + ? new File("./../../docker/docker-compose.hybrid.yml") + : new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.volumes.yml"), IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.yml") @@ -71,12 +75,16 @@ public class ThingsBoardDbInstaller extends ExternalResource { IS_REDIS_CLUSTER ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") : new File("./../../docker/docker-compose.redis.volumes.yml") - ); + )); + if (IS_HYBRID_MODE) { + composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); + } String identifier = Base58.randomString(6).toLowerCase(); String project = identifier + Base58.randomString(6).toLowerCase(); postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; + cassandraDataVolume = project + "_" + CASSANDRA_DATA_VOLUME; redisDataVolume = project + "_" + REDIS_DATA_VOLUME; redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME; @@ -91,6 +99,9 @@ public class ThingsBoardDbInstaller extends ExternalResource { env = new HashMap<>(); env.put("POSTGRES_DATA_VOLUME", postgresDataVolume); + if (IS_HYBRID_MODE) { + env.put("CASSANDRA_DATA_VOLUME", cassandraDataVolume); + } env.put("TB_LOG_VOLUME", tbLogVolume); env.put("TB_COAP_TRANSPORT_LOG_VOLUME", tbCoapTransportLogVolume); env.put("TB_LWM2M_TRANSPORT_LOG_VOLUME", tbLwm2mTransportLogVolume); @@ -119,6 +130,11 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + postgresDataVolume); dockerCompose.invokeDocker(); + if (IS_HYBRID_MODE) { + dockerCompose.withCommand("volume create " + cassandraDataVolume); + dockerCompose.invokeDocker(); + } + dockerCompose.withCommand("volume create " + tbLogVolume); dockerCompose.invokeDocker(); @@ -140,20 +156,23 @@ public class ThingsBoardDbInstaller extends ExternalResource { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); - String redisService = ""; + String additionalServices = ""; + if (IS_HYBRID_MODE) { + additionalServices += " cassandra"; + } if (IS_REDIS_CLUSTER) { for (int i = 0; i < 6; i++) { - redisService = redisService + " redis-node-" + i; + additionalServices = additionalServices + " redis-node-" + i; dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i); dockerCompose.invokeDocker(); } } else { - redisService = "redis"; + additionalServices += " redis"; dockerCompose.withCommand("volume create " + redisDataVolume); dockerCompose.invokeDocker(); } - dockerCompose.withCommand("up -d postgres " + redisService); + dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); From 0d82903a4d69dd4c42010fafdb02622e105ff702 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 19:49:27 +0300 Subject: [PATCH 4/9] docker-compose: synced with docker-compose project, improved merge experience --- docker/compose-utils.sh | 10 +++++----- docker/docker-compose.kafka.yml | 2 +- docker/docker-compose.pubsub.yml | 1 - docker/docker-compose.rabbitmq.yml | 2 +- docker/docker-create-log-folders.sh | 20 ++++++++++---------- docker/docker-install-tb.sh | 8 ++++++-- docker/docker-remove-services.sh | 4 +++- docker/docker-start-services.sh | 4 +++- docker/docker-stop-services.sh | 4 +++- docker/docker-update-service.sh | 8 ++++++-- docker/docker-upgrade-tb.sh | 18 +++++++++++++----- 11 files changed, 51 insertions(+), 30 deletions(-) diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index d60c81ec6a..28601afa3c 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -26,7 +26,7 @@ function additionalComposeArgs() { ADDITIONAL_COMPOSE_ARGS="-f docker-compose.hybrid.yml" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_ARGS @@ -55,7 +55,7 @@ function additionalComposeQueueArgs() { ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml" ;; *) - echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or confluent or aws-sqs or pubsub or rabbitmq or service-bus." >&2 + echo "Unknown Queue service TB_QUEUE_TYPE value specified in the .env file: '${TB_QUEUE_TYPE}'. Should be either 'kafka' or 'confluent' or 'aws-sqs' or 'pubsub' or 'rabbitmq' or 'service-bus'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_QUEUE_ARGS @@ -85,7 +85,7 @@ function additionalComposeCacheArgs() { CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac echo $CACHE_COMPOSE_ARGS @@ -102,7 +102,7 @@ function additionalStartupServices() { ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac @@ -115,7 +115,7 @@ function additionalStartupServices() { ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 82fbb6ed62..e6fa0c489f 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -95,4 +95,4 @@ services: env_file: - queue-kafka.env depends_on: - - kafka \ No newline at end of file + - kafka diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index c03132d730..7c122d0835 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -59,4 +59,3 @@ services: tb-vc-executor2: env_file: - queue-pubsub.env - diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index d1acc32014..1f1cf1554c 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -58,4 +58,4 @@ services: - queue-rabbitmq.env tb-vc-executor2: env_file: - - queue-rabbitmq.env \ No newline at end of file + - queue-rabbitmq.env diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 83257c0e66..8daf11c624 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -15,7 +15,7 @@ # limitations under the License. # -mkdir -p tb-node/log/ && sudo chown -R 799:799 tb-node/log/ +mkdir -p tb-node/log && sudo chown -R 799:799 tb-node/log mkdir -p tb-transports/coap/log && sudo chown -R 799:799 tb-transports/coap/log @@ -29,23 +29,23 @@ mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log -mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/ +mkdir -p tb-node/postgres && sudo chown -R 999:999 tb-node/postgres source .env CACHE="${CACHE:-redis}" case $CACHE in redis) - mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/ + mkdir -p tb-node/redis-data && sudo chown -R 1001:0 tb-node/redis-data ;; redis-cluster) - mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/ - mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/ - mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/ - mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/ - mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/ - mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/ + mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:0 tb-node/redis-cluster-data-0 + mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:0 tb-node/redis-cluster-data-1 + mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:0 tb-node/redis-cluster-data-2 + mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:0 tb-node/redis-cluster-data-3 + mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:0 tb-node/redis-cluster-data-4 + mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:0 tb-node/redis-cluster-data-5 ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac \ No newline at end of file diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 27e84c4d0b..7263fec7fa 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -50,9 +50,13 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES + docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \ + tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 36e464f45d..769150c1f3 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 7993ab425b..9e5d08add9 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index ae8a3c0b81..5b09aea204 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index a02f9e208a..027280635d 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -25,5 +25,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull $@ +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index c20543783b..260e14cf8b 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -48,8 +48,16 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 - -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES - -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull \ + tb-core1 + +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES + +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \ + tb-core1 From f0292849c7bcfd1d52912d7d0b0355df68521caf Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 21:00:16 +0300 Subject: [PATCH 5/9] docker-compose create folder tb-node/cassandra --- docker/docker-create-log-folders.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 8daf11c624..d1d0277d3c 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -31,6 +31,8 @@ mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log mkdir -p tb-node/postgres && sudo chown -R 999:999 tb-node/postgres +mkdir -p tb-node/cassandra && sudo chown -R 999:999 tb-node/cassandra + source .env CACHE="${CACHE:-redis}" case $CACHE in From d4ba0c3f26a6ac3cb7e48d3884c304ee8ee1825d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 21:25:55 +0300 Subject: [PATCH 6/9] docker-compose multiline refactor for better merge --- docker/docker-install-tb.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 7263fec7fa..9129c72796 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -51,7 +51,8 @@ ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then docker-compose \ - -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES fi docker-compose \ From 2aeb735b9569eb15c9fcf853c24b5ac5631011a9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Jun 2022 23:31:57 +0300 Subject: [PATCH 7/9] docker-compose: redis data chown -R 1001:1001 --- docker/docker-create-log-folders.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index d1d0277d3c..6224b7040b 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -37,15 +37,15 @@ source .env CACHE="${CACHE:-redis}" case $CACHE in redis) - mkdir -p tb-node/redis-data && sudo chown -R 1001:0 tb-node/redis-data + mkdir -p tb-node/redis-data && sudo chown -R 1001:1001 tb-node/redis-data ;; redis-cluster) - mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:0 tb-node/redis-cluster-data-0 - mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:0 tb-node/redis-cluster-data-1 - mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:0 tb-node/redis-cluster-data-2 - mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:0 tb-node/redis-cluster-data-3 - mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:0 tb-node/redis-cluster-data-4 - mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:0 tb-node/redis-cluster-data-5 + mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-0 + mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-1 + mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-2 + mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-3 + mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-4 + mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-5 ;; *) echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 From d828ea2e176177fe72ce1570cdc43778d04c12b9 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 24 Jun 2022 03:06:53 +0300 Subject: [PATCH 8/9] black-box-test: cleanup code --- .../thingsboard/server/msa/ContainerTestSuite.java | 11 ++++++----- .../server/msa/ThingsBoardDbInstaller.java | 6 +++++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 6802f19214..1c4b01c5d1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -77,7 +77,8 @@ public class ContainerTestSuite { } } - List composeFiles = new ArrayList<>(Arrays.asList(new File(targetDir + "docker-compose.yml"), + List composeFiles = new ArrayList<>(Arrays.asList( + new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), IS_HYBRID_MODE ? new File(targetDir + "docker-compose.hybrid.yml") @@ -85,11 +86,11 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml"), + ? new File(targetDir + "docker-compose.redis-cluster.yml") + : new File(targetDir + "docker-compose.redis.yml"), IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") - : new File("./../../docker/docker-compose.redis.volumes.yml"))); + ? new File(targetDir + "docker-compose.redis-cluster.volumes.yml") + : new File(targetDir + "docker-compose.redis.volumes.yml"))); if (IS_HYBRID_MODE) { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 08cadbceb2..b627d2787a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -21,7 +21,11 @@ import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import java.io.File; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; From 13765a4dfc0b8713441b93a7e929b53396a2ea38 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Fri, 24 Jun 2022 11:42:23 +0300 Subject: [PATCH 9/9] Reduced redundant logging --- application/src/main/conf/logback.xml | 2 + .../AnnotationComponentDiscoveryService.java | 10 ++--- application/src/main/resources/logback.xml | 39 +++++++++++-------- .../src/main/resources/thingsboard.yml | 3 ++ msa/vc-executor/src/main/conf/logback.xml | 2 + .../src/main/resources/logback.xml | 3 ++ transport/coap/src/main/conf/logback.xml | 2 + transport/coap/src/main/resources/logback.xml | 3 ++ transport/http/src/main/conf/logback.xml | 2 + transport/http/src/main/resources/logback.xml | 3 ++ transport/lwm2m/src/main/conf/logback.xml | 2 + .../lwm2m/src/main/resources/logback.xml | 3 ++ transport/mqtt/src/main/conf/logback.xml | 2 + transport/mqtt/src/main/resources/logback.xml | 3 ++ transport/snmp/src/main/conf/logback.xml | 2 + transport/snmp/src/main/resources/logback.xml | 3 ++ 16 files changed, 62 insertions(+), 22 deletions(-) diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 898128c1ff..6d2c95ee84 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java index f279a29aac..7a95bf78f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java @@ -161,20 +161,20 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe configurationDescriptor.set("nodeDefinition", node); scannedComponent.setConfigurationDescriptor(configurationDescriptor); scannedComponent.setClazz(clazzName); - log.info("Processing scanned component: {}", scannedComponent); + log.debug("Processing scanned component: {}", scannedComponent); } catch (Exception e) { log.error("Can't initialize component {}, due to {}", def.getBeanClassName(), e.getMessage(), e); throw new RuntimeException(e); } ComponentDescriptor persistedComponent = componentDescriptorService.findByClazz(TenantId.SYS_TENANT_ID, clazzName); if (persistedComponent == null) { - log.info("Persisting new component: {}", scannedComponent); + log.debug("Persisting new component: {}", scannedComponent); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); } else if (scannedComponent.equals(persistedComponent)) { - log.info("Component is already persisted: {}", persistedComponent); + log.debug("Component is already persisted: {}", persistedComponent); scannedComponent = persistedComponent; } else { - log.info("Component {} will be updated to {}", persistedComponent, scannedComponent); + log.debug("Component {} will be updated to {}", persistedComponent, scannedComponent); componentDescriptorService.deleteByClazz(TenantId.SYS_TENANT_ID, persistedComponent.getClazz()); scannedComponent.setId(persistedComponent.getId()); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); @@ -224,7 +224,7 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe @Override public void discoverComponents() { registerRuleNodeComponents(); - log.info("Found following definitions: {}", components.values()); + log.debug("Found following definitions: {}", components.values()); } @Override diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 995ebb43c4..4784ad2d43 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,26 +25,31 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 07233bb241..f6312a7659 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -428,6 +428,9 @@ cache: timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}" maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}" +#Disable this because it is not required. +spring.data.redis.repositories.enabled: false + redis: # standalone or cluster connection: diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml index d62cf2b3f5..c0f33852a6 100644 --- a/msa/vc-executor/src/main/conf/logback.xml +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -35,6 +35,8 @@ + + diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml index 572d093dde..e25a1995c1 100644 --- a/msa/vc-executor/src/main/resources/logback.xml +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -28,6 +28,9 @@ + + + diff --git a/transport/coap/src/main/conf/logback.xml b/transport/coap/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/coap/src/main/conf/logback.xml +++ b/transport/coap/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/coap/src/main/resources/logback.xml b/transport/coap/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/coap/src/main/resources/logback.xml +++ b/transport/coap/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/http/src/main/conf/logback.xml b/transport/http/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/http/src/main/conf/logback.xml +++ b/transport/http/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/http/src/main/resources/logback.xml b/transport/http/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/http/src/main/resources/logback.xml +++ b/transport/http/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/lwm2m/src/main/conf/logback.xml b/transport/lwm2m/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/lwm2m/src/main/conf/logback.xml +++ b/transport/lwm2m/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/lwm2m/src/main/resources/logback.xml b/transport/lwm2m/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/lwm2m/src/main/resources/logback.xml +++ b/transport/lwm2m/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/mqtt/src/main/conf/logback.xml b/transport/mqtt/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/mqtt/src/main/conf/logback.xml +++ b/transport/mqtt/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/mqtt/src/main/resources/logback.xml b/transport/mqtt/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/mqtt/src/main/resources/logback.xml +++ b/transport/mqtt/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/snmp/src/main/conf/logback.xml b/transport/snmp/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/snmp/src/main/conf/logback.xml +++ b/transport/snmp/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/snmp/src/main/resources/logback.xml b/transport/snmp/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/snmp/src/main/resources/logback.xml +++ b/transport/snmp/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + +