Browse Source

Merge branch 'develop/3.4' into vc-fixes

pull/6803/head
Andrew Shvayka 4 years ago
committed by GitHub
parent
commit
3155a2c02f
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 73
      application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
  2. 5
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  3. 9
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java
  4. 11
      application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java
  5. 7
      application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java
  6. 3
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java
  7. 7
      common/cluster-api/src/main/proto/queue.proto
  8. 39
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java
  9. 6
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java
  10. 7
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
  11. 31
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
  12. 5
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
  13. 18
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
  14. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java
  15. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java
  16. 3
      dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java
  17. 18
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  18. 134
      dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java
  19. 156
      dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java
  20. 3
      docker/.env
  21. 7
      docker/README.md
  22. 5
      docker/cache-redis-cluster.env
  23. 2
      docker/cache-redis.env
  24. 36
      docker/compose-utils.sh
  25. 26
      docker/docker-compose.aws-sqs.yml
  26. 8
      docker/docker-compose.confluent.yml
  27. 4
      docker/docker-compose.hybrid.yml
  28. 4
      docker/docker-compose.kafka.yml
  29. 61
      docker/docker-compose.postgres.volumes.yml
  30. 8
      docker/docker-compose.postgres.yml
  31. 58
      docker/docker-compose.redis-cluster.volumes.yml
  32. 148
      docker/docker-compose.redis-cluster.yml
  33. 27
      docker/docker-compose.redis.volumes.yml
  34. 97
      docker/docker-compose.redis.yml
  35. 81
      docker/docker-compose.volumes.yml
  36. 14
      docker/docker-compose.yml
  37. 21
      docker/docker-create-log-folders.sh
  38. 6
      docker/docker-install-tb.sh
  39. 4
      docker/docker-remove-services.sh
  40. 4
      docker/docker-start-services.sh
  41. 4
      docker/docker-stop-services.sh
  42. 6
      docker/docker-update-service.sh
  43. 8
      docker/docker-upgrade-tb.sh
  44. 3
      docker/tb-coap-transport.env
  45. 3
      docker/tb-http-transport.env
  46. 3
      docker/tb-lwm2m-transport.env
  47. 3
      docker/tb-mqtt-transport.env
  48. 2
      docker/tb-node.env
  49. 3
      docker/tb-snmp-transport.env
  50. 7
      msa/black-box-tests/README.md
  51. 13
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java
  52. 53
      msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java
  53. 12
      ui-ngx/src/app/core/http/entities-version-control.service.ts

73
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
@ -39,6 +38,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -56,6 +56,7 @@ import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START;
@ -228,10 +229,10 @@ public class EntitiesVersionControlController extends BaseController {
"}" +
MARKDOWN_CODE_BLOCK_END +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable String branch,
@PathVariable EntityType entityType,
@GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable EntityType entityType,
@PathVariable UUID externalEntityUuid,
@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -254,9 +255,9 @@ public class EntitiesVersionControlController extends BaseController {
"If specified branch does not exist - empty page data will be returned. " +
"The response structure is the same as for `listEntityVersions` API method." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable String branch,
@PathVariable EntityType entityType,
@GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable EntityType entityType,
@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -277,8 +278,8 @@ public class EntitiesVersionControlController extends BaseController {
"If specified branch does not exist - empty page data will be returned. " +
"The response format is the same as for `listEntityVersions` API method." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{branch}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listVersions(@PathVariable String branch,
@GetMapping(value = "/version", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listVersions(@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -300,11 +301,11 @@ public class EntitiesVersionControlController extends BaseController {
"Each entity item in the result has `externalId` property. " +
"Entities order will be the same as in the repository." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping("/entity/{branch}/{entityType}/{versionId}")
public DeferredResult<List<VersionedEntityInfo>> listEntitiesAtVersion(@PathVariable String branch,
@PathVariable EntityType entityType,
@GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"})
public DeferredResult<List<VersionedEntityInfo>> listEntitiesAtVersion(@PathVariable EntityType entityType,
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@PathVariable String versionId) throws Exception {
@PathVariable String versionId,
@RequestParam String branch) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType));
}
@ -314,10 +315,10 @@ public class EntitiesVersionControlController extends BaseController {
"Response type is the same as for listAllEntitiesAtVersion API method. \n" +
"Returned entities order will be the same as in the repository." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping("/entity/{branch}/{versionId}")
public DeferredResult<List<VersionedEntityInfo>> listAllEntitiesAtVersion(@PathVariable String branch,
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@PathVariable String versionId) throws Exception {
@GetMapping(value = "/entity/{versionId}", params = {"branch"})
public DeferredResult<List<VersionedEntityInfo>> listAllEntitiesAtVersion(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@PathVariable String versionId,
@RequestParam String branch) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId));
}
@ -342,10 +343,10 @@ public class EntitiesVersionControlController extends BaseController {
"Returns an object with current entity data and the one at a specific version. " +
"Entity data structure is the same as stored in a repository. " +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}")
public DeferredResult<EntityDataDiff> compareEntityDataToVersion(@PathVariable String branch,
@PathVariable EntityType entityType,
@GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"})
public DeferredResult<EntityDataDiff> compareEntityDataToVersion(@PathVariable EntityType entityType,
@PathVariable UUID internalEntityUuid,
@RequestParam String branch,
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@RequestParam String versionId) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
@ -485,33 +486,23 @@ public class EntitiesVersionControlController extends BaseController {
public DeferredResult<List<BranchInfo>> listBranches() throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
final TenantId tenantId = getTenantId();
ListenableFuture<List<String>> branches = versionControlService.listBranches(tenantId);
ListenableFuture<List<BranchInfo>> branches = versionControlService.listBranches(tenantId);
return wrapFuture(Futures.transform(branches, remoteBranches -> {
List<BranchInfo> infos = new ArrayList<>();
String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch();
if (StringUtils.isEmpty(defaultBranch)) {
if (remoteBranches.contains("main")) {
defaultBranch = "main";
} else {
defaultBranch = "master";
}
BranchInfo defaultBranch;
String defaultBranchName = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch();
if (StringUtils.isNotEmpty(defaultBranchName)) {
defaultBranch = new BranchInfo(defaultBranchName, true);
} else {
defaultBranch = remoteBranches.stream().filter(BranchInfo::isDefault).findFirst().orElse(null);
}
infos.add(new BranchInfo(defaultBranch, true));
for (String branch : remoteBranches) {
if (!branch.equals(defaultBranch)) {
infos.add(new BranchInfo(branch, false));
}
if (defaultBranch != null) {
infos.add(defaultBranch);
}
infos.addAll(remoteBranches.stream().filter(b -> !b.equals(defaultBranch))
.map(b -> new BranchInfo(b.getName(), false)).collect(Collectors.toList()));
return infos;
}, MoreExecutors.directExecutor()));
}
@Data
public static class BranchInfo {
private final String name;
private final boolean isDefault;
}
}

5
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -25,12 +25,10 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.TbStopWatch;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
@ -49,6 +47,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityLoadError;
@ -480,7 +479,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception {
public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception {
return gitServiceQueue.listBranches(tenantId);
}

9
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java

@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
@ -240,7 +241,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
}
@Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) {
public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) {
ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId);
return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()));
}
@ -382,7 +383,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
((CommitGitRequest) request).getFuture().set(commitResult);
} else if (vcResponseMsg.hasListBranchesResponse()) {
var listBranchesResponse = vcResponseMsg.getListBranchesResponse();
((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList());
((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList().stream().map(this::getBranchInfo).collect(Collectors.toList()));
} else if (vcResponseMsg.hasListEntitiesResponse()) {
var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse();
((ListEntitiesGitRequest) request).getFuture().set(
@ -439,6 +440,10 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())));
}
private BranchInfo getBranchInfo(TransportProtos.BranchInfoProto proto) {
return new BranchInfo(proto.getName(), proto.getIsDefault());
}
@SuppressWarnings("rawtypes")
@SneakyThrows
private EntityExportData toData(String data) {

11
application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java

@ -16,22 +16,21 @@
package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.ListenableFuture;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
@ -60,7 +59,7 @@ public interface EntitiesVersionControlService {
ListenableFuture<EntityDataDiff> compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception;
ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception;
ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception;
RepositorySettings getVersionControlSettings(TenantId tenantId);

7
application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java

@ -24,14 +24,15 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg;
import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import java.util.List;
@ -55,7 +56,7 @@ public interface GitVersionControlQueueService {
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId);
ListenableFuture<List<String>> listBranches(TenantId tenantId);
ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId);
ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId);

3
application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java

@ -16,10 +16,11 @@
package org.thingsboard.server.service.sync.vc.data;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import java.util.List;
public class ListBranchesGitRequest extends PendingGitRequest<List<String>> {
public class ListBranchesGitRequest extends PendingGitRequest<List<BranchInfo>> {
public ListBranchesGitRequest(TenantId tenantId) {
super(tenantId);

7
common/cluster-api/src/main/proto/queue.proto

@ -773,8 +773,13 @@ message ListEntitiesResponseMsg {
message ListBranchesRequestMsg {
}
message BranchInfoProto {
string name = 1;
bool isDefault = 2;
}
message ListBranchesResponseMsg {
repeated string branches = 1;
repeated BranchInfoProto branches = 1;
}
message EntityContentRequestMsg {

39
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.sync.vc;
import lombok.Data;
import java.util.Objects;
@Data
public class BranchInfo {
private final String name;
private final boolean isDefault;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BranchInfo that = (BranchInfo) o;
return Objects.equals(name, that.name);
}
@Override
public int hashCode() {
return Objects.hash(name);
}
}

6
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java

@ -42,6 +42,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.AddMsg;
import org.thingsboard.server.gen.transport.TransportProtos.BranchInfoProto;
import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg;
@ -332,7 +333,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) {
var branches = vcService.listBranches(ctx.getTenantId());
var branches = vcService.listBranches(ctx.getTenantId()).stream()
.map(branchInfo -> BranchInfoProto.newBuilder()
.setName(branchInfo.getName())
.setIsDefault(branchInfo.isDefault()).build()).collect(Collectors.toList());
reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches)));
}

7
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java

@ -29,8 +29,9 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.GitRepository.Diff;
@ -84,7 +85,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
repository.resetAndClean();
if (repository.listRemoteBranches().contains(branch)) {
if (repository.listRemoteBranches().contains(new BranchInfo(branch, false))) {
repository.merge(branch);
}
} catch (IOException | GitAPIException gitAPIException) {
@ -182,7 +183,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
}
@Override
public List<String> listBranches(TenantId tenantId) {
public List<BranchInfo> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId);
try {
return repository.listRemoteBranches();

31
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@ -41,10 +41,12 @@ import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
@ -58,6 +60,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.SortOrder;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod;
@ -69,7 +72,12 @@ import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.security.KeyPair;
import java.security.PublicKey;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -84,6 +92,8 @@ public class GitRepository {
@Getter
private final String directory;
private ObjectId headId;
private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) {
this.git = git;
this.settings = settings;
@ -135,8 +145,12 @@ public class GitRepository {
}
public void fetch() throws GitAPIException {
execute(git.fetch()
FetchResult result = execute(git.fetch()
.setRemoveDeletedRefs(true));
Ref head = result.getAdvertisedRef(Constants.HEAD);
if (head != null) {
this.headId = head.getObjectId();
}
}
public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
@ -162,13 +176,11 @@ public class GitRepository {
.include(branchId));
}
public List<String> listRemoteBranches() throws GitAPIException {
public List<BranchInfo> listRemoteBranches() throws GitAPIException {
return execute(git.branchList()
.setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
.filter(ref -> !ref.getName().equals(Constants.HEAD))
.map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
.map(name -> StringUtils.removeStart(name, "origin/"))
.map(this::toBranchInfo)
.distinct().collect(Collectors.toList());
}
@ -325,6 +337,13 @@ public class GitRepository {
.collect(Collectors.toList());
}
private BranchInfo toBranchInfo(Ref ref) {
String name = org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName());
String branchName = StringUtils.removeStart(name, "origin/");
boolean isDefault = this.headId != null && this.headId.equals(ref.getObjectId());
return new BranchInfo(branchName, isDefault);
}
private Commit toCommit(RevCommit revCommit) {
return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(),
revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress());

5
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java

@ -19,8 +19,9 @@ import org.eclipse.jgit.api.errors.GitAPIException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.GitRepository.Diff;
@ -57,7 +58,7 @@ public interface GitRepositoryService {
void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId);
List<BranchInfo> listBranches(TenantId tenantId);
String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException;

18
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java

@ -26,6 +26,7 @@ import org.springframework.data.domain.Sort;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
@ -122,14 +123,17 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(entityId, query);
} else {
long stepTs = query.getStartTs();
List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
while (stepTs < query.getEndTs()) {
long startTs = stepTs;
long endTs = stepTs + query.getInterval();
long endPeriod = query.getEndTs();
long startPeriod = query.getStartTs();
long step = query.getInterval();
while (startPeriod <= endPeriod) {
long startTs = startPeriod;
long endTs = Math.min(startPeriod + step, endPeriod + 1);
long ts = startTs + (endTs - startTs) / 2;
futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
stepTs = endTs;
ListenableFuture<Optional<TsKvEntry>> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation());
futures.add(aggregateTsKvEntry);
startPeriod = endTs;
}
return getTskvEntriesFuture(Futures.allAsList(futures));
}
@ -148,7 +152,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities));
}
private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures);
return Futures.transform(setFutures(entitiesFutures), entity -> {

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java

@ -91,7 +91,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries
.stream()
.map(query -> findAllAsync(tenantId, entityId, query))
.collect(Collectors.toList());
return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() {
return Futures.transform(Futures.allAsList(futures), new Function<>() {
@Nullable
@Override
public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) {

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java

@ -26,4 +26,4 @@ import java.util.List;
public interface AggregationTimeseriesDao {
ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query);
}
}

3
dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java

@ -39,9 +39,7 @@ import java.util.stream.Collectors;
public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService {
private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>();
protected static final ReentrantLock tsCreationLock = new ReentrantLock();
@Autowired
protected TsKvDictionaryRepository dictionaryRepository;
@ -96,4 +94,5 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni
}
}, service);
}
}

18
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@ -141,7 +141,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
@Override
public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
List<ListenableFuture<List<TsKvEntry>>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList());
return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() {
return Futures.transform(Futures.allAsList(futures), new Function<>() {
@Nullable
@Override
public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) {
@ -270,18 +270,20 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(tenantId, entityId, query);
} else {
long startPeriod = query.getStartTs();
long endPeriod = query.getEndTs();
long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS);
long stepTs = query.getStartTs();
List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
while (stepTs < query.getEndTs()) {
long startTs = stepTs;
long endTs = stepTs + step;
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder());
while (startPeriod <= endPeriod) {
long startTs = startPeriod;
long endTs = Math.min(startPeriod + step, endPeriod + 1);
long ts = endTs - startTs;
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder());
futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs)));
stepTs = endTs;
startPeriod = endTs;
}
ListenableFuture<List<Optional<TsKvEntry>>> future = Futures.allAsList(futures);
return Futures.transform(future, new Function<List<Optional<TsKvEntry>>, List<TsKvEntry>>() {
return Futures.transform(future, new Function<>() {
@Nullable
@Override
public List<TsKvEntry> apply(@Nullable List<Optional<TsKvEntry>> input) {

134
dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java

@ -184,6 +184,140 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2));
}
@Test
public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
saveEntries(deviceId, TS);
saveEntries(deviceId, TS + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0));
}
@Test
public void testFindByQuery_whenPeriodEqualsInterval() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 100L; i += 10L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 100L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 100000L; i += 10000L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 100000L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 80000L; i += 10000L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 80000L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) {
saveEntries(deviceId, i);
}
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) {
saveEntries(deviceId, i);
}
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1));
}
@Test
public void testFindByQueryDescOrder() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());

156
dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java

@ -0,0 +1,156 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sqlts;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.BDDMockito.willCallRealMethod;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.thingsboard.server.common.data.id.TenantId.SYS_TENANT_ID;
import static org.thingsboard.server.common.data.kv.Aggregation.COUNT;
public class AbstractChunkedAggregationTimeseriesDaoTest {
final int LIMIT = 1;
final String TEMP = "temp";
final String DESC = "DESC";
AbstractChunkedAggregationTimeseriesDao tsDao;
@Before
public void setUp() throws Exception {
tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class);
ListenableFuture<Optional<TsKvEntry>> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class)));
willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any());
willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any());
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull();
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 0, 1, getTsForReadTsKvQuery(0, 1), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuery.getKey(), 1, 2, getTsForReadTsKvQuery(1, 2), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), Integer.MAX_VALUE, 1L + Integer.MAX_VALUE, getTsForReadTsKvQuery(Integer.MAX_VALUE, 1L + Integer.MAX_VALUE), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
for (long i = 1; i <= 3000; i += 3) {
ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT);
}
}
long getTsForReadTsKvQuery(long startTs, long endTs) {
return startTs + (endTs - startTs) / 2L;
}
}

3
docker/.env

@ -1,5 +1,8 @@
TB_QUEUE_TYPE=kafka
# redis or redis-cluster
CACHE=redis
DOCKER_REPO=thingsboard
JS_EXECUTOR_DOCKER_NAME=tb-js-executor

7
docker/README.md

@ -17,6 +17,13 @@ In order to set database type change the value of `DATABASE` variable in `.env`
**NOTE**: According to the database type corresponding docker service will be deployed (see `docker-compose.postgres.yml`, `docker-compose.hybrid.yml` for details).
In order to set cache type change the value of `CACHE` variable in `.env` file to one of the following:
- `redis` - use Redis standalone cache (1 node - 1 master);
- `redis-cluster` - use Redis cluster cache (6 nodes - 3 masters, 3 slaves);
**NOTE**: According to the cache type corresponding docker service will be deployed (see `docker-compose.redis.yml`, `docker-compose.redis-cluster.yml` for details).
Execute the following command to create log folders for the services and chown of these folders to the docker container users.
To be able to change user, **chown** command is used, which requires sudo permissions (script will request password for a sudo access):

5
docker/cache-redis-cluster.env

@ -0,0 +1,5 @@
CACHE_TYPE=redis
REDIS_CONNECTION_TYPE=cluster
REDIS_NODES=redis-node-0:6379,redis-node-1:6379,redis-node-2:6379,redis-node-3:6379,redis-node-4:6379,redis-node-5:6379
REDIS_USE_DEFAULT_POOL_CONFIG=false
REDIS_PASSWORD=thingsboard

2
docker/cache-redis.env

@ -0,0 +1,2 @@
CACHE_TYPE=redis
REDIS_HOST=redis

36
docker/compose-utils.sh

@ -73,19 +73,51 @@ function additionalComposeMonitoringArgs() {
fi
}
function additionalComposeCacheArgs() {
source .env
CACHE_COMPOSE_ARGS=""
CACHE="${CACHE:-redis}"
case $CACHE in
redis)
CACHE_COMPOSE_ARGS="-f docker-compose.redis.yml"
;;
redis-cluster)
CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml"
;;
*)
echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
exit 1
esac
echo $CACHE_COMPOSE_ARGS
}
function additionalStartupServices() {
source .env
ADDITIONAL_STARTUP_SERVICES=""
case $DATABASE in
postgres)
ADDITIONAL_STARTUP_SERVICES=postgres
ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres"
;;
hybrid)
ADDITIONAL_STARTUP_SERVICES="postgres cassandra"
ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra"
;;
*)
echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2
exit 1
esac
CACHE="${CACHE:-redis}"
case $CACHE in
redis)
ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis"
;;
redis-cluster)
ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5"
;;
*)
echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
exit 1
esac
echo $ADDITIONAL_STARTUP_SERVICES
}

26
docker/docker-compose.aws-sqs.yml

@ -23,59 +23,33 @@ services:
tb-core1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-core2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-rule-engine1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-rule-engine2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
- redis
tb-mqtt-transport1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-mqtt-transport2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-http-transport1:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-http-transport2:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-coap-transport:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-lwm2m-transport:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper
tb-snmp-transport:
env_file:
- queue-aws-sqs.env
depends_on:
- zookeeper

8
docker/docker-compose.confluent.yml

@ -23,23 +23,15 @@ services:
tb-core1:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-core2:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-rule-engine1:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-rule-engine2:
env_file:
- queue-confluent.env
depends_on:
- redis
tb-mqtt-transport1:
env_file:
- queue-confluent.env

4
docker/docker-compose.hybrid.yml

@ -39,7 +39,6 @@ services:
- tb-node.hybrid.env
depends_on:
- zookeeper
- redis
- postgres
- cassandra
tb-core2:
@ -47,7 +46,6 @@ services:
- tb-node.hybrid.env
depends_on:
- zookeeper
- redis
- postgres
- cassandra
tb-rule-engine1:
@ -55,7 +53,6 @@ services:
- tb-node.hybrid.env
depends_on:
- zookeeper
- redis
- postgres
- cassandra
tb-rule-engine2:
@ -63,6 +60,5 @@ services:
- tb-node.hybrid.env
depends_on:
- zookeeper
- redis
- postgres
- cassandra

4
docker/docker-compose.kafka.yml

@ -36,25 +36,21 @@ services:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-core2:
env_file:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-rule-engine1:
env_file:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-rule-engine2:
env_file:
- queue-kafka.env
depends_on:
- kafka
- redis
tb-mqtt-transport1:
env_file:
- queue-kafka.env

61
docker/docker-compose.postgres.volumes.yml

@ -20,69 +20,8 @@ services:
postgres:
volumes:
- postgres-db-volume:/var/lib/postgresql/data
tb-core1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-core2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-coap-transport:
volumes:
- tb-coap-transport-log-volume:/var/log/tb-coap-transport
tb-lwm2m-transport:
volumes:
- tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport
tb-http-transport1:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-http-transport2:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-mqtt-transport1:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-mqtt-transport2:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-snmp-transport:
volumes:
- tb-snmp-transport-log-volume:/var/log/tb-snmp-transport
tb-vc-executor1:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
tb-vc-executor2:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
volumes:
postgres-db-volume:
external: true
name: ${POSTGRES_DATA_VOLUME}
tb-log-volume:
external: true
name: ${TB_LOG_VOLUME}
tb-coap-transport-log-volume:
external: true
name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
tb-lwm2m-transport-log-volume:
external: true
name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME}
tb-http-transport-log-volume:
external: true
name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
tb-mqtt-transport-log-volume:
external: true
name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}
tb-snmp-transport-log-volume:
external: true
name: ${TB_SNMP_TRANSPORT_LOG_VOLUME}
tb-vc-executor-log-volume:
external: true
name: ${TB_VC_EXECUTOR_LOG_VOLUME}

8
docker/docker-compose.postgres.yml

@ -31,27 +31,19 @@ services:
env_file:
- tb-node.postgres.env
depends_on:
- zookeeper
- redis
- postgres
tb-core2:
env_file:
- tb-node.postgres.env
depends_on:
- zookeeper
- redis
- postgres
tb-rule-engine1:
env_file:
- tb-node.postgres.env
depends_on:
- zookeeper
- redis
- postgres
tb-rule-engine2:
env_file:
- tb-node.postgres.env
depends_on:
- zookeeper
- redis
- postgres

58
docker/docker-compose.redis-cluster.volumes.yml

@ -0,0 +1,58 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
# Redis cluster
redis-node-0:
volumes:
- redis-cluster-data-0:/bitnami/redis/data
redis-node-1:
volumes:
- redis-cluster-data-1:/bitnami/redis/data
redis-node-2:
volumes:
- redis-cluster-data-2:/bitnami/redis/data
redis-node-3:
volumes:
- redis-cluster-data-3:/bitnami/redis/data
redis-node-4:
volumes:
- redis-cluster-data-4:/bitnami/redis/data
redis-node-5:
volumes:
- redis-cluster-data-5:/bitnami/redis/data
volumes:
redis-cluster-data-0:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_0}
redis-cluster-data-1:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_1}
redis-cluster-data-2:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_2}
redis-cluster-data-3:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_3}
redis-cluster-data-4:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_4}
redis-cluster-data-5:
external: true
name: ${REDIS_CLUSTER_DATA_VOLUME_5}

148
docker/docker-compose.redis-cluster.yml

@ -0,0 +1,148 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
# Redis cluster
redis-node-0:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-0:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-1:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-1:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-2:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-2:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-3:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-3:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-4:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-4:/bitnami/redis/data
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
redis-node-5:
image: bitnami/redis-cluster:7.0
volumes:
- ./tb-node/redis-cluster-data-5:/bitnami/redis/data
depends_on:
- redis-node-0
- redis-node-1
- redis-node-2
- redis-node-3
- redis-node-4
environment:
- 'REDIS_PASSWORD=thingsboard'
- 'REDISCLI_AUTH=thingsboard'
- 'REDIS_NODES=redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5'
- 'REDIS_CLUSTER_REPLICAS=1'
- 'REDIS_CLUSTER_CREATOR=yes'
# ThingsBoard setup to use redis-cluster
tb-core1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-core2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-rule-engine1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-rule-engine2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-mqtt-transport1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-mqtt-transport2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-http-transport1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-http-transport2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-coap-transport:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-lwm2m-transport:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-snmp-transport:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-vc-executor1:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5
tb-vc-executor2:
env_file:
- cache-redis-cluster.env
depends_on:
- redis-node-5

27
docker/docker-compose.redis.volumes.yml

@ -0,0 +1,27 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
redis:
volumes:
- redis-data:/bitnami/redis/data
volumes:
redis-data:
external: true
name: ${REDIS_DATA_VOLUME}

97
docker/docker-compose.redis.yml

@ -0,0 +1,97 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
# Redis standalone
redis:
restart: always
image: bitnami/redis:7.0
environment:
# ALLOW_EMPTY_PASSWORD is recommended only for development.
ALLOW_EMPTY_PASSWORD: "yes"
ports:
- '6379:6379'
volumes:
- ./tb-node/redis-data:/bitnami/redis/data
# ThingsBoard setup to use redis-standalone
tb-core1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-core2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-rule-engine1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-rule-engine2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-mqtt-transport1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-mqtt-transport2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-http-transport1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-http-transport2:
env_file:
- cache-redis.env
depends_on:
- redis
tb-coap-transport:
env_file:
- cache-redis.env
depends_on:
- redis
tb-lwm2m-transport:
env_file:
- cache-redis.env
depends_on:
- redis
tb-snmp-transport:
env_file:
- cache-redis.env
depends_on:
- redis
tb-vc-executor1:
env_file:
- cache-redis.env
depends_on:
- redis
tb-vc-executor2:
env_file:
- cache-redis.env
depends_on:
- redis

81
docker/docker-compose.volumes.yml

@ -0,0 +1,81 @@
#
# Copyright © 2016-2022 The Thingsboard Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
version: '2.2'
services:
tb-core1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-core2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine1:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-rule-engine2:
volumes:
- tb-log-volume:/var/log/thingsboard
tb-coap-transport:
volumes:
- tb-coap-transport-log-volume:/var/log/tb-coap-transport
tb-lwm2m-transport:
volumes:
- tb-lwm2m-transport-log-volume:/var/log/tb-lwm2m-transport
tb-http-transport1:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-http-transport2:
volumes:
- tb-http-transport-log-volume:/var/log/tb-http-transport
tb-mqtt-transport1:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-mqtt-transport2:
volumes:
- tb-mqtt-transport-log-volume:/var/log/tb-mqtt-transport
tb-snmp-transport:
volumes:
- tb-snmp-transport-log-volume:/var/log/tb-snmp-transport
tb-vc-executor1:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
tb-vc-executor2:
volumes:
- tb-vc-executor-log-volume:/var/log/tb-vc-executor
volumes:
tb-log-volume:
external: true
name: ${TB_LOG_VOLUME}
tb-coap-transport-log-volume:
external: true
name: ${TB_COAP_TRANSPORT_LOG_VOLUME}
tb-lwm2m-transport-log-volume:
external: true
name: ${TB_LWM2M_TRANSPORT_LOG_VOLUME}
tb-http-transport-log-volume:
external: true
name: ${TB_HTTP_TRANSPORT_LOG_VOLUME}
tb-mqtt-transport-log-volume:
external: true
name: ${TB_MQTT_TRANSPORT_LOG_VOLUME}
tb-snmp-transport-log-volume:
external: true
name: ${TB_SNMP_TRANSPORT_LOG_VOLUME}
tb-vc-executor-log-volume:
external: true
name: ${TB_VC_EXECUTOR_LOG_VOLUME}

14
docker/docker-compose.yml

@ -20,21 +20,17 @@ version: '2.2'
services:
zookeeper:
restart: always
image: "zookeeper:3.5"
image: "zookeeper:3.8.0"
ports:
- "2181"
environment:
ZOO_MY_ID: 1
ZOO_SERVERS: server.1=zookeeper:2888:3888;zookeeper:2181
redis:
restart: always
image: redis:4.0
ports:
- "6379"
ZOO_ADMINSERVER_ENABLED: "false"
tb-js-executor:
restart: always
image: "${DOCKER_REPO}/${JS_EXECUTOR_DOCKER_NAME}:${TB_VERSION}"
scale: 20
scale: 10
env_file:
- tb-js-executor.env
tb-core1:
@ -59,7 +55,6 @@ services:
- ./tb-node/log:/var/log/thingsboard
depends_on:
- zookeeper
- redis
- tb-js-executor
- tb-rule-engine1
- tb-rule-engine2
@ -85,7 +80,6 @@ services:
- ./tb-node/log:/var/log/thingsboard
depends_on:
- zookeeper
- redis
- tb-js-executor
- tb-rule-engine1
- tb-rule-engine2
@ -109,7 +103,6 @@ services:
- ./tb-node/log:/var/log/thingsboard
depends_on:
- zookeeper
- redis
- tb-js-executor
tb-rule-engine2:
restart: always
@ -131,7 +124,6 @@ services:
- ./tb-node/log:/var/log/thingsboard
depends_on:
- zookeeper
- redis
- tb-js-executor
tb-mqtt-transport1:
restart: always

21
docker/docker-create-log-folders.sh

@ -28,3 +28,24 @@ mkdir -p tb-transports/mqtt/log && sudo chown -R 799:799 tb-transports/mqtt/log
mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log
mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log
mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/
source .env
CACHE="${CACHE:-redis}"
case $CACHE in
redis)
mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/
;;
redis-cluster)
mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/
mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/
mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/
mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/
mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/
mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/
;;
*)
echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2
exit 1
esac

6
docker/docker-install-tb.sh

@ -45,12 +45,14 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
fi
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1

4
docker/docker-remove-services.sh

@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v

4
docker/docker-start-services.sh

@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d

4
docker/docker-stop-services.sh

@ -23,6 +23,8 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop

6
docker/docker-update-service.sh

@ -23,5 +23,7 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@

8
docker/docker-upgrade-tb.sh

@ -44,10 +44,12 @@ ADDITIONAL_COMPOSE_QUEUE_ARGS=$(additionalComposeQueueArgs) || exit $?
ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $?
ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $?
ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $?
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d redis $ADDITIONAL_STARTUP_SERVICES
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES
docker-compose -f docker-compose.yml $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1
docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1

3
docker/tb-coap-transport.env

@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

3
docker/tb-http-transport.env

@ -7,6 +7,3 @@ HTTP_REQUEST_TIMEOUT=60000
METRICS_ENABLED=true
METRICS_ENDPOINTS_EXPOSE=prometheus
CACHE_TYPE=redis
REDIS_HOST=redis

3
docker/tb-lwm2m-transport.env

@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

3
docker/tb-mqtt-transport.env

@ -10,6 +10,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

2
docker/tb-node.env

@ -4,8 +4,6 @@ ZOOKEEPER_ENABLED=true
ZOOKEEPER_URL=zookeeper:2181
JS_EVALUATOR=remote
TRANSPORT_TYPE=remote
CACHE_TYPE=redis
REDIS_HOST=redis
HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false

3
docker/tb-snmp-transport.env

@ -6,6 +6,3 @@ METRICS_ENDPOINTS_EXPOSE=prometheus
WEB_APPLICATION_ENABLE=true
WEB_APPLICATION_TYPE=servlet
HTTP_BIND_PORT=8081
CACHE_TYPE=redis
REDIS_HOST=redis

7
msa/black-box-tests/README.md

@ -18,8 +18,13 @@ As result, in REPOSITORY column, next images should be present:
thingsboard/tb-web-ui
thingsboard/tb-js-executor
- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory:
- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis standalone:
mvn clean install -DblackBoxTests.skip=false
- Run the black box tests in the [msa/black-box-tests](../black-box-tests) directory with Redis cluster:
mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.redisCluster=true

13
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java

@ -39,7 +39,7 @@ import static org.junit.Assert.fail;
@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"})
@Slf4j
public class ContainerTestSuite {
final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster"));
private static final String SOURCE_DIR = "./../../docker/";
private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*";
private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*";
@ -52,6 +52,7 @@ public class ContainerTestSuite {
@ClassRule
public static DockerComposeContainer getTestContainer() {
if (testContainer == null) {
log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER);
boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers"));
try {
final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/";
@ -73,9 +74,17 @@ public class ContainerTestSuite {
testContainer = new DockerComposeContainerImpl<>(
new File(targetDir + "docker-compose.yml"),
new File(targetDir + "docker-compose.volumes.yml"),
new File(targetDir + "docker-compose.postgres.yml"),
new File(targetDir + "docker-compose.postgres.volumes.yml"),
new File(targetDir + "docker-compose.kafka.yml"))
new File(targetDir + "docker-compose.kafka.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.yml")
: new File("./../../docker/docker-compose.redis.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.volumes.yml")
: new File("./../../docker/docker-compose.redis.volumes.yml")
)
.withPull(false)
.withLocalCompose(true)
.withTailChildContainers(!skipTailChildContainers)

53
msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.msa;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.junit.rules.ExternalResource;
import org.testcontainers.utility.Base58;
@ -24,10 +25,16 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@Slf4j
public class ThingsBoardDbInstaller extends ExternalResource {
final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster"));
private final static String POSTGRES_DATA_VOLUME = "tb-postgres-test-data-volume";
private final static String REDIS_DATA_VOLUME = "tb-redis-data-volume";
private final static String REDIS_CLUSTER_DATA_VOLUME = "tb-redis-cluster-data-volume";
private final static String TB_LOG_VOLUME = "tb-log-test-volume";
private final static String TB_COAP_TRANSPORT_LOG_VOLUME = "tb-coap-transport-log-test-volume";
private final static String TB_LWM2M_TRANSPORT_LOG_VOLUME = "tb-lwm2m-transport-log-test-volume";
@ -39,6 +46,9 @@ public class ThingsBoardDbInstaller extends ExternalResource {
private final DockerComposeExecutor dockerCompose;
private final String postgresDataVolume;
private final String redisDataVolume;
private final String redisClusterDataVolume;
private final String tbLogVolume;
private final String tbCoapTransportLogVolume;
private final String tbLwm2mTransportLogVolume;
@ -49,14 +59,26 @@ public class ThingsBoardDbInstaller extends ExternalResource {
private final Map<String, String> env;
public ThingsBoardDbInstaller() {
List<File> composeFiles = Arrays.asList(new File("./../../docker/docker-compose.yml"),
log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER);
List<File> composeFiles = Arrays.asList(
new File("./../../docker/docker-compose.yml"),
new File("./../../docker/docker-compose.volumes.yml"),
new File("./../../docker/docker-compose.postgres.yml"),
new File("./../../docker/docker-compose.postgres.volumes.yml"));
new File("./../../docker/docker-compose.postgres.volumes.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.yml")
: new File("./../../docker/docker-compose.redis.yml"),
IS_REDIS_CLUSTER
? new File("./../../docker/docker-compose.redis-cluster.volumes.yml")
: new File("./../../docker/docker-compose.redis.volumes.yml")
);
String identifier = Base58.randomString(6).toLowerCase();
String project = identifier + Base58.randomString(6).toLowerCase();
postgresDataVolume = project + "_" + POSTGRES_DATA_VOLUME;
redisDataVolume = project + "_" + REDIS_DATA_VOLUME;
redisClusterDataVolume = project + "_" + REDIS_CLUSTER_DATA_VOLUME;
tbLogVolume = project + "_" + TB_LOG_VOLUME;
tbCoapTransportLogVolume = project + "_" + TB_COAP_TRANSPORT_LOG_VOLUME;
tbLwm2mTransportLogVolume = project + "_" + TB_LWM2M_TRANSPORT_LOG_VOLUME;
@ -76,6 +98,13 @@ public class ThingsBoardDbInstaller extends ExternalResource {
env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume);
env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume);
env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume);
if (IS_REDIS_CLUSTER) {
for (int i = 0; i < 6; i++) {
env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i);
}
} else {
env.put("REDIS_DATA_VOLUME", redisDataVolume);
}
dockerCompose.withEnv(env);
}
@ -111,7 +140,20 @@ public class ThingsBoardDbInstaller extends ExternalResource {
dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume);
dockerCompose.invokeDocker();
dockerCompose.withCommand("up -d redis postgres");
String redisService = "";
if (IS_REDIS_CLUSTER) {
for (int i = 0; i < 6; i++) {
redisService = redisService + " redis-node-" + i;
dockerCompose.withCommand("volume create " + redisClusterDataVolume + '-' + i);
dockerCompose.invokeDocker();
}
} else {
redisService = "redis";
dockerCompose.withCommand("volume create " + redisDataVolume);
dockerCompose.invokeDocker();
}
dockerCompose.withCommand("up -d postgres " + redisService);
dockerCompose.invokeCompose();
dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1");
@ -137,7 +179,10 @@ public class ThingsBoardDbInstaller extends ExternalResource {
dockerCompose.withCommand("volume rm -f " + postgresDataVolume + " " + tbLogVolume +
" " + tbCoapTransportLogVolume + " " + tbLwm2mTransportLogVolume + " " + tbHttpTransportLogVolume +
" " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume);
" " + tbMqttTransportLogVolume + " " + tbSnmpTransportLogVolume + " " + tbVcExecutorLogVolume +
(IS_REDIS_CLUSTER
? IntStream.range(0, 6).mapToObj(i -> " " + redisClusterDataVolume + '-' + i).collect(Collectors.joining())
: redisDataVolume));
dockerCompose.invokeDocker();
}

12
ui-ngx/src/app/core/http/entities-version-control.service.ts

@ -118,20 +118,23 @@ export class EntitiesVersionControlService {
public listEntityVersions(pageLink: PageLink, branch: string,
externalEntityId: EntityId,
config?: RequestConfig): Observable<PageData<EntityVersion>> {
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${branch}/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}`,
const encodedBranch = encodeURIComponent(branch);
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${externalEntityId.entityType}/${externalEntityId.id}${pageLink.toQuery()}&branch=${encodedBranch}`,
defaultHttpOptionsFromConfig(config));
}
public listEntityTypeVersions(pageLink: PageLink, branch: string,
entityType: EntityType,
config?: RequestConfig): Observable<PageData<EntityVersion>> {
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${branch}/${entityType}${pageLink.toQuery()}`,
const encodedBranch = encodeURIComponent(branch);
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${entityType}${pageLink.toQuery()}&branch=${encodedBranch}`,
defaultHttpOptionsFromConfig(config));
}
public listVersions(pageLink: PageLink, branch: string,
config?: RequestConfig): Observable<PageData<EntityVersion>> {
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version/${branch}${pageLink.toQuery()}`,
const encodedBranch = encodeURIComponent(branch);
return this.http.get<PageData<EntityVersion>>(`/api/entities/vc/version${pageLink.toQuery()}&branch=${encodedBranch}`,
defaultHttpOptionsFromConfig(config));
}
@ -160,7 +163,8 @@ export class EntitiesVersionControlService {
entityId: EntityId,
versionId: string,
config?: RequestConfig): Observable<EntityDataDiff> {
return this.http.get<EntityDataDiff>(`/api/entities/vc/diff/${branch}/${entityId.entityType}/${entityId.id}?versionId=${versionId}`,
const encodedBranch = encodeURIComponent(branch);
return this.http.get<EntityDataDiff>(`/api/entities/vc/diff/${entityId.entityType}/${entityId.id}?branch=${encodedBranch}&versionId=${versionId}`,
defaultHttpOptionsFromConfig(config));
}

Loading…
Cancel
Save