diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 10928f1b51..798556998c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -29,15 +29,18 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.exporting.data.EntityExportData; import org.thingsboard.server.service.sync.importing.EntityImportResult; import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; import org.thingsboard.server.service.sync.vcs.data.EntityVersion; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; @RestController @RequestMapping("/api/entities/vc") @@ -51,13 +54,27 @@ public class EntitiesVersionControlController extends BaseController { @PostMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, + @PathVariable("entityId") UUID id, @RequestParam String branch, @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); + EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); } + @PostMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, + @RequestParam UUID[] ids, + @RequestParam String branch, + @RequestBody String versionName) throws Exception { + List entitiesIds = Arrays.stream(ids) + .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) + .collect(Collectors.toList()); + return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); + } + + + @GetMapping("/version/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public List listEntityVersions(@PathVariable EntityType entityType, @@ -67,29 +84,68 @@ public class EntitiesVersionControlController extends BaseController { return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); } + @GetMapping("/version/{entityType}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listEntityTypeVersions(@PathVariable EntityType entityType, + @RequestParam String branch) throws Exception { + return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); + } + + @GetMapping("/version") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listVersions(@RequestParam String branch) throws Exception { + return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); + } + + + + @GetMapping("/files/version/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List listFilesAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); + } + @GetMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId); + return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); } @PostMapping("/entity/{entityType}/{entityId}/{versionId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, @PathVariable("entityId") UUID entityUuid, + @RequestParam String branch, @PathVariable String versionId) throws Exception { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId); + EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); + onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + return result; + } + + @PostMapping("/entity/{versionId}") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List>> loadAllAtVersion(@RequestParam String branch, + @PathVariable String versionId) throws Exception { + SecurityUser user = getCurrentUser(); + List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); + resultList.forEach(result -> { + onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); + }); + return resultList; } @GetMapping("/branches") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") public Set getAllowedBranches() throws ThingsboardException { return versionControlService.getAllowedBranches(getTenantId()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java index 0d78496f16..32da1309fa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.eclipse.jgit.api.errors.GitAPIException; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; @@ -46,6 +47,8 @@ import org.thingsboard.server.utils.git.Repository; import org.thingsboard.server.utils.git.data.Commit; import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -54,7 +57,9 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @Service @@ -72,8 +77,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private static final String SETTINGS_KEY = "vc"; private Repository repository; - private final ReentrantLock fetchLock = new ReentrantLock(); - private final Lock writeLock = new ReentrantLock(); + private final Lock fetchLock = new ReentrantLock(); + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); @AfterStartUp public void init() throws Exception { @@ -89,17 +94,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - public void fetch() throws Exception { + private void fetch() throws Exception { if (repository == null) return; - - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); } @@ -118,20 +115,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportOutboundRelations(false) .build(); List>> entityDataList = entitiesIds.stream() - .map(entityId -> { - return exportImportService.exportEntity(tenantId, entityId, exportSettings); - }) + .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) .collect(Collectors.toList()); - if (fetchLock.tryLock()) { - try { - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } + tryFetch(); - writeLock.lock(); + repositoryLock.writeLock().lock(); try { if (repository.listBranches().contains(branch)) { repository.checkout(branch); @@ -148,9 +137,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); repository.push(); - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + return toVersion(commit); } finally { - writeLock.unlock(); + repositoryLock.writeLock().unlock(); } } @@ -158,38 +147,72 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); } @Override public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); + return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); + } - return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream() - .map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName())) - .collect(Collectors.toList()); + @Override + public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { + return listVersions(tenantId, branch, null, limit); + } + + private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, limit).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + + } finally { + repositoryLock.readLock().unlock(); + } } @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - checkRepository(); - // FIXME [viacheslav]: validate access + public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + return repository.listFilesAtCommit(versionId); + } finally { + repositoryLock.readLock().unlock(); + } + } - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + + + @Override + public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() + .noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Unknown version"); + } + + String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); + return parseEntityData(entityDataJson); + } finally { + repositoryLock.readLock().unlock(); + } } + @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, versionId); + public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { + EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() .importInboundRelations(false) .importOutboundRelations(false) @@ -197,6 +220,47 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .build()); } + @Override + public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + repositoryLock.readLock().lock(); + try { + List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() + .map(entityDataFilePath -> { + String entityDataJson; + try { + entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + return parseEntityData(entityDataJson); + }) + .collect(Collectors.toList()); + + return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() + .importInboundRelations(false) + .importOutboundRelations(false) + .updateReferencesToOtherEntities(true) + .build()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + private void tryFetch() throws GitAPIException { + repositoryLock.readLock().lock(); + try { + if (fetchLock.tryLock()) { + try { + log.info("Fetching remote repository"); + repository.fetch(); + } finally { + fetchLock.unlock(); + } + } + } finally { + repositoryLock.readLock().unlock(); + } + } private String getRelativePathForEntity(EntityId entityId) { @@ -210,6 +274,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont private void checkBranch(TenantId tenantId, String branch) { + // TODO [viacheslav]: all branches are available by default? if (!getAllowedBranches(tenantId).contains(branch)) { throw new IllegalArgumentException("Tenant does not have access to this branch"); } @@ -222,9 +287,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .orElse(Collections.emptySet()); } + private EntityVersion toVersion(Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); + } + + private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { + return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); + } + + + @Override public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - this.repository = initRepository(settings.getGitSettings()); + repositoryLock.writeLock().lock(); + try { + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); + } AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) .orElseGet(() -> { @@ -244,7 +324,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } - private void checkRepository() { if (repository == null) { throw new IllegalStateException("Repository is not initialized"); @@ -263,12 +342,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } public void resetRepository() throws Exception { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; + repositoryLock.writeLock().lock(); + try { + if (this.repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + this.repository = null; + } + EntitiesVersionControlSettings settings = getSettings(); + this.repository = initRepository(settings.getGitSettings()); + } finally { + repositoryLock.writeLock().unlock(); } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java index a9a3c0e3e8..e05ad17826 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java @@ -37,10 +37,17 @@ public interface EntitiesVersionControlService { List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; + List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception; - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception; + List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; + + List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; void saveSettings(EntitiesVersionControlSettings settings) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java index 8099be8259..23eb7d7db4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/git/Repository.java @@ -113,13 +113,13 @@ public class Repository { } - public List listFilesAtCommit(Commit commit) throws IOException { - return listFilesAtCommit(commit, null); + public List listFilesAtCommit(String commitId) throws IOException { + return listFilesAtCommit(commitId, null); } - public List listFilesAtCommit(Commit commit, String path) throws IOException { + public List listFilesAtCommit(String commitId, String path) throws IOException { List files = new ArrayList<>(); - RevCommit revCommit = resolveCommit(commit.getId()); + RevCommit revCommit = resolveCommit(commitId); try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) { treeWalk.reset(revCommit.getTree().getId()); if (StringUtils.isNotEmpty(path)) {