Browse Source

Merge branch 'develop/3.4' into vc-fixes

pull/6803/head
Andrew Shvayka 4 years ago
committed by GitHub
parent
commit
3155a2c02f
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 73
      application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
  2. 5
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  3. 9
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java
  4. 11
      application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java
  5. 7
      application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java
  6. 3
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java
  7. 7
      common/cluster-api/src/main/proto/queue.proto
  8. 39
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java
  9. 6
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java
  10. 7
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
  11. 31
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
  12. 5
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
  13. 18
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
  14. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java
  15. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java
  16. 3
      dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java
  17. 18
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  18. 134
      dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java
  19. 156
      dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java
  20. 3
      docker/.env
  21. 7
      docker/README.md
  22. 5
      docker/cache-redis-cluster.env
  23. 2
      docker/cache-redis.env
  24. 36
      docker/compose-utils.sh
  25. 26
      docker/docker-compose.aws-sqs.yml
  26. 8
      docker/docker-compose.confluent.yml
  27. 4
      docker/docker-compose.hybrid.yml
  28. 4
      docker/docker-compose.kafka.yml
  29. 61
      docker/docker-compose.postgres.volumes.yml
  30. 8
      docker/docker-compose.postgres.yml
  31. 58
      docker/docker-compose.redis-cluster.volumes.yml
  32. 148
      docker/docker-compose.redis-cluster.yml
  33. 27
      docker/docker-compose.redis.volumes.yml
  34. 97
      docker/docker-compose.redis.yml
  35. 81
      docker/docker-compose.volumes.yml
  36. 14
      docker/docker-compose.yml
  37. 21
      docker/docker-create-log-folders.sh
  38. 6
      docker/docker-install-tb.sh
  39. 4
      docker/docker-remove-services.sh
  40. 4
      docker/docker-start-services.sh
  41. 4
      docker/docker-stop-services.sh
  42. 6
      docker/docker-update-service.sh
  43. 8
      docker/docker-upgrade-tb.sh
  44. 3
      docker/tb-coap-transport.env
  45. 3
      docker/tb-http-transport.env
  46. 3
      docker/tb-lwm2m-transport.env
  47. 3
      docker/tb-mqtt-transport.env
  48. 2
      docker/tb-node.env
  49. 3
      docker/tb-snmp-transport.env
  50. 7
      msa/black-box-tests/README.md
  51. 13
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java
  52. 53
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java
  53. 12
      ui-ngx/src/app/core/http/entities-version-control.service.ts

73
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam; import io.swagger.annotations.ApiParam;
import lombok.Data;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.GetMapping;
@ -39,6 +38,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -56,6 +56,7 @@ import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.UUID; import java.util.UUID;
import java.util.stream.Collectors;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START;
@ -228,10 +229,10 @@ public class EntitiesVersionControlController extends BaseController {
"}" + "}" +
MARKDOWN_CODE_BLOCK_END + MARKDOWN_CODE_BLOCK_END +
TENANT_AUTHORITY_PARAGRAPH) TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"}) @GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable String branch, public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable EntityType entityType,
@PathVariable EntityType entityType,
@PathVariable UUID externalEntityUuid, @PathVariable UUID externalEntityUuid,
@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize, @RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -254,9 +255,9 @@ public class EntitiesVersionControlController extends BaseController {
"If specified branch does not exist - empty page data will be returned. " + "If specified branch does not exist - empty page data will be returned. " +
"The response structure is the same as for `listEntityVersions` API method." + "The response structure is the same as for `listEntityVersions` API method." +
TENANT_AUTHORITY_PARAGRAPH) TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"}) @GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable String branch, public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable EntityType entityType,
@PathVariable EntityType entityType, @RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize, @RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -277,8 +278,8 @@ public class EntitiesVersionControlController extends BaseController {
"If specified branch does not exist - empty page data will be returned. " + "If specified branch does not exist - empty page data will be returned. " +
"The response format is the same as for `listEntityVersions` API method." + "The response format is the same as for `listEntityVersions` API method." +
TENANT_AUTHORITY_PARAGRAPH) TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{branch}", params = {"pageSize", "page"}) @GetMapping(value = "/version", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listVersions(@PathVariable String branch, public DeferredResult<PageData<EntityVersion>> listVersions(@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize, @RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -300,11 +301,11 @@ public class EntitiesVersionControlController extends BaseController {
"Each entity item in the result has `externalId` property. " + "Each entity item in the result has `externalId` property. " +
"Entities order will be the same as in the repository." + "Entities order will be the same as in the repository." +
TENANT_AUTHORITY_PARAGRAPH) TENANT_AUTHORITY_PARAGRAPH)
@GetMapping("/entity/{branch}/{entityType}/{versionId}") @GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"})
public DeferredResult<List<VersionedEntityInfo>> listEntitiesAtVersion(@PathVariable String branch, public DeferredResult<List<VersionedEntityInfo>> listEntitiesAtVersion(@PathVariable EntityType entityType,
@PathVariable EntityType entityType,
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@PathVariable String versionId) throws Exception { @PathVariable String versionId,
@RequestParam String branch) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType));
} }
@ -314,10 +315,10 @@ public class EntitiesVersionControlController extends BaseController {
"Response type is the same as for listAllEntitiesAtVersion API method. \n" + "Response type is the same as for listAllEntitiesAtVersion API method. \n" +
"Returned entities order will be the same as in the repository." + "Returned entities order will be the same as in the repository." +
TENANT_AUTHORITY_PARAGRAPH) TENANT_AUTHORITY_PARAGRAPH)
@GetMapping("/entity/{branch}/{versionId}") @GetMapping(value = "/entity/{versionId}", params = {"branch"})
public DeferredResult<List<VersionedEntityInfo>> listAllEntitiesAtVersion(@PathVariable String branch, public DeferredResult<List<VersionedEntityInfo>> listAllEntitiesAtVersion(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId,
@PathVariable String versionId) throws Exception { @RequestParam String branch) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId));
} }
@ -342,10 +343,10 @@ public class EntitiesVersionControlController extends BaseController {
"Returns an object with current entity data and the one at a specific version. " + "Returns an object with current entity data and the one at a specific version. " +
"Entity data structure is the same as stored in a repository. " + "Entity data structure is the same as stored in a repository. " +
TENANT_AUTHORITY_PARAGRAPH) TENANT_AUTHORITY_PARAGRAPH)
@GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}") @GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"})
public DeferredResult<EntityDataDiff> compareEntityDataToVersion(@PathVariable String branch, public DeferredResult<EntityDataDiff> compareEntityDataToVersion(@PathVariable EntityType entityType,
@PathVariable EntityType entityType,
@PathVariable UUID internalEntityUuid, @PathVariable UUID internalEntityUuid,
@RequestParam String branch,
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@RequestParam String versionId) throws Exception { @RequestParam String versionId) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
@ -485,33 +486,23 @@ public class EntitiesVersionControlController extends BaseController {
public DeferredResult<List<BranchInfo>> listBranches() throws Exception { public DeferredResult<List<BranchInfo>> listBranches() throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
final TenantId tenantId = getTenantId(); final TenantId tenantId = getTenantId();
ListenableFuture<List<String>> branches = versionControlService.listBranches(tenantId); ListenableFuture<List<BranchInfo>> branches = versionControlService.listBranches(tenantId);
return wrapFuture(Futures.transform(branches, remoteBranches -> { return wrapFuture(Futures.transform(branches, remoteBranches -> {
List<BranchInfo> infos = new ArrayList<>(); List<BranchInfo> infos = new ArrayList<>();
BranchInfo defaultBranch;
String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch(); String defaultBranchName = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch();
if (StringUtils.isEmpty(defaultBranch)) { if (StringUtils.isNotEmpty(defaultBranchName)) {
if (remoteBranches.contains("main")) { defaultBranch = new BranchInfo(defaultBranchName, true);
defaultBranch = "main"; } else {
} else { defaultBranch = remoteBranches.stream().filter(BranchInfo::isDefault).findFirst().orElse(null);
defaultBranch = "master";
}
} }
infos.add(new BranchInfo(defaultBranch, true)); if (defaultBranch != null) {
infos.add(defaultBranch);
for (String branch : remoteBranches) {
if (!branch.equals(defaultBranch)) {
infos.add(new BranchInfo(branch, false));
}
} }
infos.addAll(remoteBranches.stream().filter(b -> !b.equals(defaultBranch))
.map(b -> new BranchInfo(b.getName(), false)).collect(Collectors.toList()));
return infos; return infos;
}, MoreExecutors.directExecutor())); }, MoreExecutors.directExecutor()));
} }
@Data
public static class BranchInfo {
private final String name;
private final boolean isDefault;
}
} }

5
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -25,12 +25,10 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate; import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.DonAsynchron; import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.common.util.TbStopWatch;
import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.ExportableEntity;
@ -49,6 +47,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityLoadError; import org.thingsboard.server.common.data.sync.vc.EntityLoadError;
@ -480,7 +479,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override @Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception { public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception {
return gitServiceQueue.listBranches(tenantId); return gitServiceQueue.listBranches(tenantId);
} }

9
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java

@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff; import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
@ -240,7 +241,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
} }
@Override @Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) { public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) {
ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId); ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId);
return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build())); return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()));
} }
@ -382,7 +383,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
((CommitGitRequest) request).getFuture().set(commitResult); ((CommitGitRequest) request).getFuture().set(commitResult);
} else if (vcResponseMsg.hasListBranchesResponse()) { } else if (vcResponseMsg.hasListBranchesResponse()) {
var listBranchesResponse = vcResponseMsg.getListBranchesResponse(); var listBranchesResponse = vcResponseMsg.getListBranchesResponse();
((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList()); ((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList().stream().map(this::getBranchInfo).collect(Collectors.toList()));
} else if (vcResponseMsg.hasListEntitiesResponse()) { } else if (vcResponseMsg.hasListEntitiesResponse()) {
var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse(); var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse();
((ListEntitiesGitRequest) request).getFuture().set( ((ListEntitiesGitRequest) request).getFuture().set(
@ -439,6 +440,10 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB()))); return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())));
} }
private BranchInfo getBranchInfo(TransportProtos.BranchInfoProto proto) {
return new BranchInfo(proto.getName(), proto.getIsDefault());
}
@SuppressWarnings("rawtypes") @SuppressWarnings("rawtypes")
@SneakyThrows @SneakyThrows
private EntityExportData toData(String data) { private EntityExportData toData(String data) {

11
application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java

@ -16,22 +16,21 @@
package org.thingsboard.server.service.sync.vc; package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff; import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo; import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult; import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
@ -60,7 +59,7 @@ public interface EntitiesVersionControlService {
ListenableFuture<EntityDataDiff> compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception; ListenableFuture<EntityDataDiff> compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception;
ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception; ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception;
RepositorySettings getVersionControlSettings(TenantId tenantId); RepositorySettings getVersionControlSettings(TenantId tenantId);

7
application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java

@ -24,14 +24,15 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg;
import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import java.util.List; import java.util.List;
@ -55,7 +56,7 @@ public interface GitVersionControlQueueService {
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId);
ListenableFuture<List<String>> listBranches(TenantId tenantId); ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId);
ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId); ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId);

3
application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java

@ -16,10 +16,11 @@
package org.thingsboard.server.service.sync.vc.data; package org.thingsboard.server.service.sync.vc.data;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import java.util.List; import java.util.List;
public class ListBranchesGitRequest extends PendingGitRequest<List<String>> { public class ListBranchesGitRequest extends PendingGitRequest<List<BranchInfo>> {
public ListBranchesGitRequest(TenantId tenantId) { public ListBranchesGitRequest(TenantId tenantId) {
super(tenantId); super(tenantId);

7
common/cluster-api/src/main/proto/queue.proto

@ -773,8 +773,13 @@ message ListEntitiesResponseMsg {
message ListBranchesRequestMsg { message ListBranchesRequestMsg {
} }
message BranchInfoProto {
string name = 1;
bool isDefault = 2;
}
message ListBranchesResponseMsg { message ListBranchesResponseMsg {
repeated string branches = 1; repeated BranchInfoProto branches = 1;
} }
message EntityContentRequestMsg { message EntityContentRequestMsg {

39
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.sync.vc;
import lombok.Data;
import java.util.Objects;
@Data
public class BranchInfo {
private final String name;
private final boolean isDefault;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BranchInfo that = (BranchInfo) o;
return Objects.equals(name, that.name);
}
@Override
public int hashCode() {
return Objects.hash(name);
}
}

6
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java

@ -42,6 +42,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.AddMsg; import org.thingsboard.server.gen.transport.TransportProtos.AddMsg;
import org.thingsboard.server.gen.transport.TransportProtos.BranchInfoProto;
import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg; import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg;
@ -332,7 +333,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
} }
private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) { private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) {
var branches = vcService.listBranches(ctx.getTenantId()); var branches = vcService.listBranches(ctx.getTenantId()).stream()
.map(branchInfo -> BranchInfoProto.newBuilder()
.setName(branchInfo.getName())
.setIsDefault(branchInfo.isDefault()).build()).collect(Collectors.toList());
reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches))); reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches)));
} }

7
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java

@ -29,8 +29,9 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import org.thingsboard.server.service.sync.vc.GitRepository.Diff;
@ -84,7 +85,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch()); repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
repository.resetAndClean(); repository.resetAndClean();
if (repository.listRemoteBranches().contains(branch)) { if (repository.listRemoteBranches().contains(new BranchInfo(branch, false))) {
repository.merge(branch); repository.merge(branch);
} }
} catch (IOException | GitAPIException gitAPIException) { } catch (IOException | GitAPIException gitAPIException) {
@ -182,7 +183,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
} }
@Override @Override
public List<String> listBranches(TenantId tenantId) { public List<BranchInfo> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId); GitRepository repository = checkRepository(tenantId);
try { try {
return repository.listRemoteBranches(); return repository.listRemoteBranches();

31
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@ -41,10 +41,12 @@ import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk; import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider; import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.transport.RefSpec; import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.SshTransport; import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
@ -58,6 +60,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.SortOrder; import org.thingsboard.server.common.data.page.SortOrder;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod; import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod;
@ -69,7 +72,12 @@ import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets; import java.nio.charset.StandardCharsets;
import java.security.KeyPair; import java.security.KeyPair;
import java.security.PublicKey; import java.security.PublicKey;
import java.util.*; import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function; import java.util.function.Function;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -84,6 +92,8 @@ public class GitRepository {
@Getter @Getter
private final String directory; private final String directory;
private ObjectId headId;
private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) { private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) {
this.git = git; this.git = git;
this.settings = settings; this.settings = settings;
@ -135,8 +145,12 @@ public class GitRepository {
} }
public void fetch() throws GitAPIException { public void fetch() throws GitAPIException {
execute(git.fetch() FetchResult result = execute(git.fetch()
.setRemoveDeletedRefs(true)); .setRemoveDeletedRefs(true));
Ref head = result.getAdvertisedRef(Constants.HEAD);
if (head != null) {
this.headId = head.getObjectId();
}
} }
public void deleteLocalBranchIfExists(String branch) throws GitAPIException { public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
@ -162,13 +176,11 @@ public class GitRepository {
.include(branchId)); .include(branchId));
} }
public List<BranchInfo> listRemoteBranches() throws GitAPIException {
public List<String> listRemoteBranches() throws GitAPIException {
return execute(git.branchList() return execute(git.branchList()
.setListMode(ListBranchCommand.ListMode.REMOTE)).stream() .setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
.filter(ref -> !ref.getName().equals(Constants.HEAD)) .filter(ref -> !ref.getName().equals(Constants.HEAD))
.map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName())) .map(this::toBranchInfo)
.map(name -> StringUtils.removeStart(name, "origin/"))
.distinct().collect(Collectors.toList()); .distinct().collect(Collectors.toList());
} }
@ -325,6 +337,13 @@ public class GitRepository {
.collect(Collectors.toList()); .collect(Collectors.toList());
} }
private BranchInfo toBranchInfo(Ref ref) {
String name = org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName());
String branchName = StringUtils.removeStart(name, "origin/");
boolean isDefault = this.headId != null && this.headId.equals(ref.getObjectId());
return new BranchInfo(branchName, isDefault);
}
private Commit toCommit(RevCommit revCommit) { private Commit toCommit(RevCommit revCommit) {
return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(), return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(),
revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress()); revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress());

5
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java

@ -19,8 +19,9 @@ import org.eclipse.jgit.api.errors.GitAPIException;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion; import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.GitRepository.Diff; import org.thingsboard.server.service.sync.vc.GitRepository.Diff;
@ -57,7 +58,7 @@ public interface GitRepositoryService {
void abort(PendingCommit commit); void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId); List<BranchInfo> listBranches(TenantId tenantId);
String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException;

18
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java

@ -26,6 +26,7 @@ import org.springframework.data.domain.Sort;
import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation; import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvEntry;
@ -122,14 +123,17 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
if (query.getAggregation() == Aggregation.NONE) { if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(entityId, query); return findAllAsyncWithLimit(entityId, query);
} else { } else {
long stepTs = query.getStartTs();
List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>(); List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
while (stepTs < query.getEndTs()) { long endPeriod = query.getEndTs();
long startTs = stepTs; long startPeriod = query.getStartTs();
long endTs = stepTs + query.getInterval(); long step = query.getInterval();
while (startPeriod <= endPeriod) {
long startTs = startPeriod;
long endTs = Math.min(startPeriod + step, endPeriod + 1);
long ts = startTs + (endTs - startTs) / 2; long ts = startTs + (endTs - startTs) / 2;
futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation())); ListenableFuture<Optional<TsKvEntry>> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation());
stepTs = endTs; futures.add(aggregateTsKvEntry);
startPeriod = endTs;
} }
return getTskvEntriesFuture(Futures.allAsList(futures)); return getTskvEntriesFuture(Futures.allAsList(futures));
} }
@ -148,7 +152,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities)); return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities));
} }
private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) { ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>(); List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures); switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures);
return Futures.transform(setFutures(entitiesFutures), entity -> { return Futures.transform(setFutures(entitiesFutures), entity -> {

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java

@ -91,7 +91,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries
.stream() .stream()
.map(query -> findAllAsync(tenantId, entityId, query)) .map(query -> findAllAsync(tenantId, entityId, query))
.collect(Collectors.toList()); .collect(Collectors.toList());
return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() { return Futures.transform(Futures.allAsList(futures), new Function<>() {
@Nullable @Nullable
@Override @Override
public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) { public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) {

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java

@ -26,4 +26,4 @@ import java.util.List;
public interface AggregationTimeseriesDao { public interface AggregationTimeseriesDao {
ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query); ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query);
} }

3
dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java

@ -39,9 +39,7 @@ import java.util.stream.Collectors;
public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService { public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService {
private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>(); private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>();
protected static final ReentrantLock tsCreationLock = new ReentrantLock(); protected static final ReentrantLock tsCreationLock = new ReentrantLock();
@Autowired @Autowired
protected TsKvDictionaryRepository dictionaryRepository; protected TsKvDictionaryRepository dictionaryRepository;
@ -96,4 +94,5 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni
} }
}, service); }, service);
} }
} }

18
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@ -141,7 +141,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
@Override @Override
public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) { public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
List<ListenableFuture<List<TsKvEntry>>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList()); List<ListenableFuture<List<TsKvEntry>>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList());
return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() { return Futures.transform(Futures.allAsList(futures), new Function<>() {
@Nullable @Nullable
@Override @Override
public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) { public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) {
@ -270,18 +270,20 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
if (query.getAggregation() == Aggregation.NONE) { if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(tenantId, entityId, query); return findAllAsyncWithLimit(tenantId, entityId, query);
} else { } else {
long startPeriod = query.getStartTs();
long endPeriod = query.getEndTs();
long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS); long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS);
long stepTs = query.getStartTs();
List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>(); List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
while (stepTs < query.getEndTs()) { while (startPeriod <= endPeriod) {
long startTs = stepTs; long startTs = startPeriod;
long endTs = stepTs + step; long endTs = Math.min(startPeriod + step, endPeriod + 1);
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder()); long ts = endTs - startTs;
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder());
futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs))); futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs)));
stepTs = endTs; startPeriod = endTs;
} }
ListenableFuture<List<Optional<TsKvEntry>>> future = Futures.allAsList(futures); ListenableFuture<List<Optional<TsKvEntry>>> future = Futures.allAsList(futures);
return Futures.transform(future, new Function<List<Optional<TsKvEntry>>, List<TsKvEntry>>() { return Futures.transform(future, new Function<>() {
@Nullable @Nullable
@Override @Override
public List<TsKvEntry> apply(@Nullable List<Optional<TsKvEntry>> input) { public List<TsKvEntry> apply(@Nullable List<Optional<TsKvEntry>> input) {

134
dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java

@ -184,6 +184,140 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2)); Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2));
} }
@Test
public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
saveEntries(deviceId, TS);
saveEntries(deviceId, TS + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0));
}
@Test
public void testFindByQuery_whenPeriodEqualsInterval() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 100L; i += 10L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 100L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 100000L; i += 10000L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 100000L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 80000L; i += 10000L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 80000L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) {
saveEntries(deviceId, i);
}
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) {
saveEntries(deviceId, i);
}
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1));
}
@Test @Test
public void testFindByQueryDescOrder() throws Exception { public void testFindByQueryDescOrder() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased()); DeviceId deviceId = new DeviceId(Uuids.timeBased());

156
dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java

@ -0,0 +1,156 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sqlts;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.BDDMockito.willCallRealMethod;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.thingsboard.server.common.data.id.TenantId.SYS_TENANT_ID;
import static org.thingsboard.server.common.data.kv.Aggregation.COUNT;
public class AbstractChunkedAggregationTimeseriesDaoTest {
final int LIMIT = 1;
final String TEMP = "temp";
final String DESC = "DESC";
AbstractChunkedAggregationTimeseriesDao tsDao;
@Before
public void setUp() throws Exception {
tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class);
ListenableFuture<Optional<TsKvEntry>> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class)));
willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any());
willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any());
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull();
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 0, 1, getTsForReadTsKvQuery(0, 1), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuery.getKey(), 1, 2, getTsForReadTsKvQuery(1, 2), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), Integer.MAX_VALUE, 1L + Integer.MAX_VALUE, getTsForReadTsKvQuery(Integer.MAX_VALUE, 1L + Integer.MAX_VALUE), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
for (long i = 1; i <= 3000; i += 3) {
ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT);
}
}
long getTsForReadTsKvQuery(long startTs, long endTs) {
return startTs + (endTs - startTs) / 2L;
}
}

3
docker/.env

@ -1,5 +1,8 @@
TB_QUEUE_TYPE=kafka TB_QUEUE_TYPE=kafka
# redis or redis-cluster
CACHE=redis
DOCKER_REPO=thingsboard DOCKER_REPO=thingsboard
JS_EXECUTOR_DOCKER_NAME=tb-js-executor JS_EXECUTOR_DOCKER_NAME=tb-js-executor

7
docker/README.md

@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env`
**NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details). **NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details).
In order to set cache type change the value of `CACHE` variable in `.env` file to one of the following:
- `redis` - use Redis standalone cache (1 node - 1 master);
- `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves);
**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details).
Execute the following command to create log folders for the services and chown of these folders to the docker container users. Execute the following command to create log folders for the services and chown of these folders to the docker container users.
To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access): To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access):

5
docker/cache-redis-cluster.env

@ -0,0 +1,5 @@
CACHE_TYPE=redis
REDIS_CONNECTION_TYPE=cluster
REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379
REDIS_USE_DEFAULT_POOL_CONFIG=false
REDIS_PASSWORD=thingsboard

2
docker/cache-redis.env

@ -0,0 +1,2 @@
CACHE_TYPE=redis
REDIS_HOST=redis

36
docker/compose-utils.sh

@ -73,19 +73,51 @@ function additionalComposeMonitoringArgs() {
fi fi
} }
function additionalComposeCacheArgs() {
source .env
CACHE_COMPOSE_ARGS=""
CACHE="${CACHE:-redis}"
case $CACHE in
redis)
CACHE_COMPOSE_ARGS="-f docker-compose.redis.yml"
;;
redis-cluster)
CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml"
;;
*)
echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
exit 1
esac
echo $CACHE_COMPOSE_ARGS
}
function additionalStartupServices() { function additionalStartupServices() {
source .env source .env
ADDITIONAL_STARTUP_SERVICES="" ADDITIONAL_STARTUP_SERVICES=""
case $DATABASE in case $DATABASE in
postgres) postgres)
ADDITIONAL_STARTUP_SERVICES=postgres ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres"
;; ;;
hybrid) hybrid)
ADDITIONAL_STARTUP_SERVICES="postgres cassandra" ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra"
;; ;;
*) *)
echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2
exit 1 exit 1
esac esac
CACHE="${CACHE:-redis}"
case $CACHE in
redis)
ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis"
;;
redis-cluster)
ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
;;
*)
echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
exit 1
esac
echo $ADDITIONAL_STARTUP_SERVICES echo $ADDITIONAL_STARTUP_SERVICES
} }

26
docker/docker-compose.aws-sqs.yml

@ -23,59 +23,33 @@ services:
tb-core1: tb-core1:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-core2: tb-core2:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-rule-engine1: tb-rule-engine1:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-rule-engine2: tb-rule-engine2:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-mqtt-transport1: tb-mqtt-transport1:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
tb-mqtt-transport2: tb-mqtt-transport2:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
tb-http-transport1: tb-http-transport1:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
tb-http-transport2: tb-http-transport2:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
tb-coap-transport: tb-coap-transport:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
tb-lwm2m-transport: tb-lwm2m-transport:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper
tb-snmp-transport: tb-snmp-transport:
env_file: env_file:
- queue-aws-sqs.env - queue-aws-sqs.env
depends_on:
- zookeeper

8
docker/docker-compose.confluent.yml

@ -23,23 +23,15 @@ services:
tb-core1: tb-core1:
env_file: env_file:
- queue-confluent.env - queue-confluent.env
depends_on:
- redis
tb-core2: tb-core2:
env_file: env_file:
- queue-confluent.env - queue-confluent.env
depends_on:
- redis
tb-rule-engine1: tb-rule-engine1:
env_file: env_file:
- queue-confluent.env - queue-confluent.env
depends_on:
- redis
tb-rule-engine2: tb-rule-engine2:
env_file: env_file:
- queue-confluent.env - queue-confluent.env
depends_on:
- redis
tb-mqtt-transport1: tb-mqtt-transport1:
env_file: env_file:
- queue-confluent.env - queue-confluent.env

4
docker/docker-compose.hybrid.yml

@ -39,7 +39,6 @@ services:
- tb-node.hybrid.env - tb-node.hybrid.env
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- postgres - postgres
- cassandra - cassandra
tb-core2: tb-core2:
@ -47,7 +46,6 @@ services:
- tb-node.hybrid.env - tb-node.hybrid.env
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- postgres - postgres
- cassandra - cassandra
tb-rule-engine1: tb-rule-engine1:
@ -55,7 +53,6 @@ services:
- tb-node.hybrid.env - tb-node.hybrid.env
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- postgres - postgres
- cassandra - cassandra
tb-rule-engine2: tb-rule-engine2:
@ -63,6 +60,5 @@ services:
- tb-node.hybrid.env - tb-node.hybrid.env
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- postgres - postgres
- cassandra - cassandra

4
docker/docker-compose.kafka.yml

@ -36,25 +36,21 @@ services:
- queue-kafka.env - queue-kafka.env
depends_on: depends_on:
- kafka - kafka
- redis
tb-core2: tb-core2:
env_file: env_file:
- queue-kafka.env - queue-kafka.env
depends_on: depends_on:
- kafka - kafka
- redis
tb-rule-engine1: tb-rule-engine1:
env_file: env_file:
- queue-kafka.env - queue-kafka.env
depends_on: depends_on:
- kafka - kafka
- redis
tb-rule-engine2: tb-rule-engine2:
env_file: env_file:
- queue-kafka.env - queue-kafka.env
depends_on: depends_on:
- kafka - kafka
- redis
tb-mqtt-transport1: tb-mqtt-transport1:
env_file: env_file:
- queue-kafka.env - queue-kafka.env

61
docker/docker-compose.postgres.volumes.yml

@ -20,69 +20,8 @@ services:
postgres: postgres:
volumes: volumes:
- postgres-db-volume:/var/lib/postgresql/data - postgres-db-volume:/var/lib/postgresql/data
tb-core1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-core2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-coap-transport:
volumes:
- tb-coap-transport-log-volume:/var/log/tb-coap-transport
tb-lwm2m-transport:
volumes:
- tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport
tb-http-transport1:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-http-transport2:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-mqtt-transport1:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-mqtt-transport2:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-snmp-transport:
volumes:
- tb-snmp-transport-log-volume:/var/log/tb-snmp-transport
tb-vc-executor1:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
tb-vc-executor2:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
volumes: volumes:
postgres-db-volume: postgres-db-volume:
external: true external: true
name: ${POSTGRES_DATA_VOLUME} name: ${POSTGRES_DATA_VOLUME}
tb-log-volume:
external: true
name: ${TB_LOG_VOLUME}
tb-coap-transport-log-volume:
external: true
name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
tb-lwm2m-transport-log-volume:
external: true
name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME}
tb-http-transport-log-volume:
external: true
name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
tb-mqtt-transport-log-volume:
external: true
name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}
tb-snmp-transport-log-volume:
external: true
name: ${TB_SNMP_TRANSPORT_LOG_VOLUME}
tb-vc-executor-log-volume:
external: true
name: ${TB_VC_EXECUTOR_LOG_VOLUME}

8
docker/docker-compose.postgres.yml

@ -31,27 +31,19 @@ services:
env_file: env_file:
- tb-node.postgres.env - tb-node.postgres.env
depends_on: depends_on:
- zookeeper
- redis
- postgres - postgres
tb-core2: tb-core2:
env_file: env_file:
- tb-node.postgres.env - tb-node.postgres.env
depends_on: depends_on:
- zookeeper
- redis
- postgres - postgres
tb-rule-engine1: tb-rule-engine1:
env_file: env_file:
- tb-node.postgres.env - tb-node.postgres.env
depends_on: depends_on:
- zookeeper
- redis
- postgres - postgres
tb-rule-engine2: tb-rule-engine2:
env_file: env_file:
- tb-node.postgres.env - tb-node.postgres.env
depends_on: depends_on:
- zookeeper
- redis
- postgres - postgres

58
docker/docker-compose.redis-cluster.volumes.yml

@ -0,0 +1,58 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
# Redis cluster
redis-node-0:
volumes:
- redis-cluster-data-0:/bitnami/redis/data
redis-node-1:
volumes:
- redis-cluster-data-1:/bitnami/redis/data
redis-node-2:
volumes:
- redis-cluster-data-2:/bitnami/redis/data
redis-node-3:
volumes:
- redis-cluster-data-3:/bitnami/redis/data
redis-node-4:
volumes:
- redis-cluster-data-4:/bitnami/redis/data
redis-node-5:
volumes:
- redis-cluster-data-5:/bitnami/redis/data
volumes:
redis-cluster-data-0:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_0}
redis-cluster-data-1:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_1}
redis-cluster-data-2:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_2}
redis-cluster-data-3:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_3}
redis-cluster-data-4:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_4}
redis-cluster-data-5:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_5}

148
docker/docker-compose.redis-cluster.yml

@ -0,0 +1,148 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
# Redis cluster
redis-node-0:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-0:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-1:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-1:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-2:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-2:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-3:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-3:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-4:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-4:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-5:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-5:/bitnami/redis/data
depends_on:
- redis-node-0
- redis-node-1
- redis-node-2
- redis-node-3
- redis-node-4
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
- 'REDIS_CLUSTER_REPLICAS=1'
- 'REDIS_CLUSTER_CREATOR=yes'
# ThingsBoard setup to use redis-cluster
tb-core1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-core2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-rule-engine1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-rule-engine2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-mqtt-transport1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-mqtt-transport2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-http-transport1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-http-transport2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-coap-transport:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-lwm2m-transport:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-snmp-transport:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-vc-executor1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-vc-executor2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5

27
docker/docker-compose.redis.volumes.yml

@ -0,0 +1,27 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
redis:
volumes:
- redis-data:/bitnami/redis/data
volumes:
redis-data:
external: true
name: ${REDIS_DATA_VOLUME}

97
docker/docker-compose.redis.yml

@ -0,0 +1,97 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
# Redis standalone
redis:
restart: always
image: bitnami/redis:7.0
environment:
# ALLOW_EMPTY_PASSWORD is recommended only for development.
ALLOW_EMPTY_PASSWORD: "yes"
ports:
- '6379:6379'
volumes:
- ./tb-node/redis-data:/bitnami/redis/data
# ThingsBoard setup to use redis-standalone
tb-core1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-core2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-rule-engine1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-rule-engine2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-mqtt-transport1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-mqtt-transport2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-http-transport1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-http-transport2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-coap-transport:
env_file:
- cache-redis.env
depends_on:
- redis
tb-lwm2m-transport:
env_file:
- cache-redis.env
depends_on:
- redis
tb-snmp-transport:
env_file:
- cache-redis.env
depends_on:
- redis
tb-vc-executor1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-vc-executor2:
env_file:
- cache-redis.env
depends_on:
- redis

81
docker/docker-compose.volumes.yml

@ -0,0 +1,81 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
tb-core1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-core2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-coap-transport:
volumes:
- tb-coap-transport-log-volume:/var/log/tb-coap-transport
tb-lwm2m-transport:
volumes:
- tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport
tb-http-transport1:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-http-transport2:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-mqtt-transport1:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-mqtt-transport2:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-snmp-transport:
volumes:
- tb-snmp-transport-log-volume:/var/log/tb-snmp-transport
tb-vc-executor1:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
tb-vc-executor2:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
volumes:
tb-log-volume:
external: true
name: ${TB_LOG_VOLUME}
tb-coap-transport-log-volume:
external: true
name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
tb-lwm2m-transport-log-volume:
external: true
name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME}
tb-http-transport-log-volume:
external: true
name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
tb-mqtt-transport-log-volume:
external: true
name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}
tb-snmp-transport-log-volume:
external: true
name: ${TB_SNMP_TRANSPORT_LOG_VOLUME}
tb-vc-executor-log-volume:
external: true
name: ${TB_VC_EXECUTOR_LOG_VOLUME}

14
docker/docker-compose.yml

@ -20,21 +20,17 @@ version: '2.2'
services: services:
zookeeper: zookeeper:
restart: always restart: always
image: "zookeeper:3.5" image: "zookeeper:3.8.0"
ports: ports:
- "2181" - "2181"
environment: environment:
ZOO_MY_ID: 1 ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181 ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181
redis: ZOO_ADMINSERVER_ENABLED: "false"
restart: always
image: redis:4.0
ports:
- "6379"
tb-js-executor: tb-js-executor:
restart: always restart: always
image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}" image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}"
scale: 20 scale: 10
env_file: env_file:
- tb-js-executor.env - tb-js-executor.env
tb-core1: tb-core1:
@ -59,7 +55,6 @@ services:
- ./tb-node/log:/var/log/thingsboard - ./tb-node/log:/var/log/thingsboard
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- tb-js-executor - tb-js-executor
- tb-rule-engine1 - tb-rule-engine1
- tb-rule-engine2 - tb-rule-engine2
@ -85,7 +80,6 @@ services:
- ./tb-node/log:/var/log/thingsboard - ./tb-node/log:/var/log/thingsboard
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- tb-js-executor - tb-js-executor
- tb-rule-engine1 - tb-rule-engine1
- tb-rule-engine2 - tb-rule-engine2
@ -109,7 +103,6 @@ services:
- ./tb-node/log:/var/log/thingsboard - ./tb-node/log:/var/log/thingsboard
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- tb-js-executor - tb-js-executor
tb-rule-engine2: tb-rule-engine2:
restart: always restart: always
@ -131,7 +124,6 @@ services:
- ./tb-node/log:/var/log/thingsboard - ./tb-node/log:/var/log/thingsboard
depends_on: depends_on:
- zookeeper - zookeeper
- redis
- tb-js-executor - tb-js-executor
tb-mqtt-transport1: tb-mqtt-transport1:
restart: always restart: always

21
docker/docker-create-log-folders.sh

@ -28,3 +28,24 @@ mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log
mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log
mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log
mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/
source .env
CACHE="${CACHE:-redis}"
case $CACHE in
redis)
mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/
;;
redis-cluster)
mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/
mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/
mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/
mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/
mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/
mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/
;;
*)
echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
exit 1
esac

6
docker/docker-install-tb.sh

@ -45,12 +45,14 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
fi fi
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1

4
docker/docker-remove-services.sh

@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v

4
docker/docker-start-services.sh

@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d

4
docker/docker-stop-services.sh

@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop

6
docker/docker-update-service.sh

@ -23,5 +23,7 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@

8
docker/docker-upgrade-tb.sh

@ -44,10 +44,12 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1

3
docker/tb-coap-transport.env

@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081 HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

3
docker/tb-http-transport.env

@ -7,6 +7,3 @@ HTTP_REQUEST_TIMEOUT=60000
METRICS_ENABLED=true METRICS_ENABLED=true
METRICS_ENDPOINTS_EXPOSE=prometheus METRICS_ENDPOINTS_EXPOSE=prometheus
CACHE_TYPE=redis
REDIS_HOST=redis

3
docker/tb-lwm2m-transport.env

@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081 HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

3
docker/tb-mqtt-transport.env

@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081 HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

2
docker/tb-node.env

@ -4,8 +4,6 @@ ZOOKEEPER_ENABLED=true
ZOOKEEPER_URL=zookeeper:2181 ZOOKEEPER_URL=zookeeper:2181
JS_EVALUATOR=remote JS_EVALUATOR=remote
TRANSPORT_TYPE=remote TRANSPORT_TYPE=remote
CACHE_TYPE=redis
REDIS_HOST=redis
HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false

3
docker/tb-snmp-transport.env

@ -6,6 +6,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081 HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

7
msa/black-box-tests/README.md

@ -18,8 +18,13 @@ As result, in REPOSITORY column, next images should be present:
thingsboard/tb-web-ui thingsboard/tb-web-ui
thingsboard/tb-js-executor thingsboard/tb-js-executor
- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory: - Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone:
mvn clean install -DblackBoxTests.skip=false mvn clean install -DblackBoxTests.skip=false
- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis cluster:
mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true

13
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@ -39,7 +39,7 @@ import static org.junit.Assert.fail;
@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) @ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"})
@Slf4j @Slf4j
public class ContainerTestSuite { public class ContainerTestSuite {
final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster"));
private static final String SOURCE_DIR = "./../../docker/"; private static final String SOURCE_DIR = "./../../docker/";
private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*";
private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*";
@ -52,6 +52,7 @@ public class ContainerTestSuite {
@ClassRule @ClassRule
public static DockerComposeContainer getTestContainer() { public static DockerComposeContainer getTestContainer() {
if (testContainer == null) { if (testContainer == null) {
log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER);
boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers"));
try { try {
final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/";
@ -73,9 +74,17 @@ public class ContainerTestSuite {
testContainer = new DockerComposeContainerImpl<>( testContainer = new DockerComposeContainerImpl<>(
new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.yml"),
new File(targetDir + "docker-compose.volumes.yml"),
new File(targetDir + "docker-compose.postgres.yml"), new File(targetDir + "docker-compose.postgres.yml"),
new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.postgres.volumes.yml"),
new File(targetDir + "docker-compose.kafka.yml")) new File(targetDir + "docker-compose.kafka.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.yml")
: new File("./../../docker/docker-compose.redis.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.volumes.yml")
: new File("./../../docker/docker-compose.redis.volumes.yml")
)
.withPull(false) .withPull(false)
.withLocalCompose(true) .withLocalCompose(true)
.withTailChildContainers(!skipTailChildContainers) .withTailChildContainers(!skipTailChildContainers)

53
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java

@ -15,6 +15,7 @@
*/ */
package org.thingsboard.server.msa; package org.thingsboard.server.msa;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomStringUtils;
import org.junit.rules.ExternalResource; import org.junit.rules.ExternalResource;
import org.testcontainers.utility.Base58; import org.testcontainers.utility.Base58;
@ -24,10 +25,16 @@ import java.util.Arrays;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@Slf4j
public class ThingsBoardDbInstaller extends ExternalResource { public class ThingsBoardDbInstaller extends ExternalResource {
final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster"));
private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume"; private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume";
private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume";
private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume";
private final static String TB_LOG_VOLUME = "tb-log-test-volume"; private final static String TB_LOG_VOLUME = "tb-log-test-volume";
private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume"; private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume";
private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume"; private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume";
@ -39,6 +46,9 @@ public class ThingsBoardDbInstaller extends ExternalResource {
private final DockerComposeExecutor dockerCompose; private final DockerComposeExecutor dockerCompose;
private final String postgresDataVolume; private final String postgresDataVolume;
private final String redisDataVolume;
private final String redisClusterDataVolume;
private final String tbLogVolume; private final String tbLogVolume;
private final String tbCoapTransportLogVolume; private final String tbCoapTransportLogVolume;
private final String tbLwm2mTransportLogVolume; private final String tbLwm2mTransportLogVolume;
@ -49,14 +59,26 @@ public class ThingsBoardDbInstaller extends ExternalResource {
private final Map<String, String> env; private final Map<String, String> env;
public ThingsBoardDbInstaller() { public ThingsBoardDbInstaller() {
List<File> composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"), log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER);
List<File> composeFiles = Arrays.asList(
new File("./../../docker/docker-compose.yml"),
new File("./../../docker/docker-compose.volumes.yml"),
new File("./../../docker/docker-compose.postgres.yml"), new File("./../../docker/docker-compose.postgres.yml"),
new File("./../../docker/docker-compose.postgres.volumes.yml")); new File("./../../docker/docker-compose.postgres.volumes.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.yml")
: new File("./../../docker/docker-compose.redis.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.volumes.yml")
: new File("./../../docker/docker-compose.redis.volumes.yml")
);
String identifier = Base58.randomString(6).toLowerCase(); String identifier = Base58.randomString(6).toLowerCase();
String project = identifier + Base58.randomString(6).toLowerCase(); String project = identifier + Base58.randomString(6).toLowerCase();
postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME; postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME;
redisDataVolume = project + "_" + REDIS_DATA_VOLUME;
redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME;
tbLogVolume = project + "_" + TB_LOG_VOLUME; tbLogVolume = project + "_" + TB_LOG_VOLUME;
tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME; tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME;
tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME; tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME;
@ -76,6 +98,13 @@ public class ThingsBoardDbInstaller extends ExternalResource {
env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume);
env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume);
env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume);
if (IS_REDIS_CLUSTER) {
for (int i = 0; i < 6; i++) {
env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i);
}
} else {
env.put("REDIS_DATA_VOLUME", redisDataVolume);
}
dockerCompose.withEnv(env); dockerCompose.withEnv(env);
} }
@ -111,7 +140,20 @@ public class ThingsBoardDbInstaller extends ExternalResource {
dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume);
dockerCompose.invokeDocker(); dockerCompose.invokeDocker();
dockerCompose.withCommand("up -d redis postgres"); String redisService = "";
if (IS_REDIS_CLUSTER) {
for (int i = 0; i < 6; i++) {
redisService = redisService + " redis-node-" + i;
dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i);
dockerCompose.invokeDocker();
}
} else {
redisService = "redis";
dockerCompose.withCommand("volume create " + redisDataVolume);
dockerCompose.invokeDocker();
}
dockerCompose.withCommand("up -d postgres " + redisService);
dockerCompose.invokeCompose(); dockerCompose.invokeCompose();
dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1");
@ -137,7 +179,10 @@ public class ThingsBoardDbInstaller extends ExternalResource {
dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume + dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume +
" " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume + " " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume +
" " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume); " " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume +
(IS_REDIS_CLUSTER
? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining())
: redisDataVolume));
dockerCompose.invokeDocker(); dockerCompose.invokeDocker();
} }

12
ui-ngx/src/app/core/http/entities-version-control.service.ts

@ -118,20 +118,23 @@ export class EntitiesVersionControlService {
public listEntityVersions(pageLink: PageLink, branch: string, public listEntityVersions(pageLink: PageLink, branch: string,
externalEntityId: EntityId, externalEntityId: EntityId,
config?: RequestConfig): Observable<PageData<EntityVersion>> { config?: RequestConfig): Observable<PageData<EntityVersion>> {
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`, const encodedBranch = encodeURIComponent(branch);
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}&branch=${encodedBranch}`,
defaultHttpOptionsFromConfig(config)); defaultHttpOptionsFromConfig(config));
} }
public listEntityTypeVersions(pageLink: PageLink, branch: string, public listEntityTypeVersions(pageLink: PageLink, branch: string,
entityType: EntityType, entityType: EntityType,
config?: RequestConfig): Observable<PageData<EntityVersion>> { config?: RequestConfig): Observable<PageData<EntityVersion>> {
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`, const encodedBranch = encodeURIComponent(branch);
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${entityType}${pageLink.toQuery()}&branch=${encodedBranch}`,
defaultHttpOptionsFromConfig(config)); defaultHttpOptionsFromConfig(config));
} }
public listVersions(pageLink: PageLink, branch: string, public listVersions(pageLink: PageLink, branch: string,
config?: RequestConfig): Observable<PageData<EntityVersion>> { config?: RequestConfig): Observable<PageData<EntityVersion>> {
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`, const encodedBranch = encodeURIComponent(branch);
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version${pageLink.toQuery()}&branch=${encodedBranch}`,
defaultHttpOptionsFromConfig(config)); defaultHttpOptionsFromConfig(config));
} }
@ -160,7 +163,8 @@ export class EntitiesVersionControlService {
entityId: EntityId, entityId: EntityId,
versionId: string, versionId: string,
config?: RequestConfig): Observable<EntityDataDiff> { config?: RequestConfig): Observable<EntityDataDiff> {
return this.http.get<EntityDataDiff>(`/api/entities/vc/diff/${branch}/${entityId.entityType}/${entityId.id}?versionId=${versionId}`, const encodedBranch = encodeURIComponent(branch);
return this.http.get<EntityDataDiff>(`/api/entities/vc/diff/${entityId.entityType}/${entityId.id}?branch=${encodedBranch}&versionId=${versionId}`,
defaultHttpOptionsFromConfig(config)); defaultHttpOptionsFromConfig(config));
} }

Loading…
Cancel
Save