tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

API for listing entities at version, loading all entities at version, listing all available versions

pull/6558/head
Viacheslav Klimov 4 years ago
parent
f82be0153b
commit
b3dfed5bad
4 changed files with 207 additions and 60 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 64
      application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
  2. 184
      application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java
  3. 11
      application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java
  4. 8
      application/src/main/java/org/thingsboard/server/utils/git/Repository.java

64
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
View File

@ -29,15 +29,18 @@ import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.exporting.data.EntityExportData;
import org.thingsboard.server.service.sync.importing.EntityImportResult;
import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService;
import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings;
import org.thingsboard.server.service.sync.vcs.data.EntityVersion;
import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;
@RestController
@RequestMapping("/api/entities/vc")
@ -51,13 +54,27 @@ public class EntitiesVersionControlController extends BaseController {
@PostMapping("/version/{entityType}/{entityId}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public EntityVersion saveEntityVersion(@PathVariable EntityType entityType,
@PathVariable("entityId") UUID entityUuid,
@PathVariable("entityId") UUID id,
@RequestParam String branch,
@RequestBody String versionName) throws Exception {
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id);
return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName);
}
@PostMapping("/version/{entityType}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType,
@RequestParam UUID[] ids,
@RequestParam String branch,
@RequestBody String versionName) throws Exception {
List<EntityId> entitiesIds = Arrays.stream(ids)
.map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id))
.collect(Collectors.toList());
return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName);
}
@GetMapping("/version/{entityType}/{entityId}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public List<EntityVersion> listEntityVersions(@PathVariable EntityType entityType,
@ -67,29 +84,68 @@ public class EntitiesVersionControlController extends BaseController {
return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE);
}
@GetMapping("/version/{entityType}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public List<EntityVersion> listEntityTypeVersions(@PathVariable EntityType entityType,
@RequestParam String branch) throws Exception {
return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE);
}
@GetMapping("/version")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public List<EntityVersion> listVersions(@RequestParam String branch) throws Exception {
return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE);
}
@GetMapping("/files/version/{versionId}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public List<String> listFilesAtVersion(@RequestParam String branch,
@PathVariable String versionId) throws Exception {
return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId);
}
@GetMapping("/entity/{entityType}/{entityId}/{versionId}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public EntityExportData<ExportableEntity<EntityId>> getEntityAtVersion(@PathVariable EntityType entityType,
@PathVariable("entityId") UUID entityUuid,
@RequestParam String branch,
@PathVariable String versionId) throws Exception {
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
return versionControlService.getEntityAtVersion(getTenantId(), entityId, versionId);
return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId);
}
@PostMapping("/entity/{entityType}/{entityId}/{versionId}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public EntityImportResult<ExportableEntity<EntityId>> loadEntityVersion(@PathVariable EntityType entityType,
@PathVariable("entityId") UUID entityUuid,
@RequestParam String branch,
@PathVariable String versionId) throws Exception {
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid);
return versionControlService.loadEntityVersion(getTenantId(), entityId, versionId);
EntityImportResult<ExportableEntity<EntityId>> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId);
onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null);
return result;
}
@PostMapping("/entity/{versionId}")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(@RequestParam String branch,
@PathVariable String versionId) throws Exception {
SecurityUser user = getCurrentUser();
List<EntityImportResult<ExportableEntity<EntityId>>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId);
resultList.forEach(result -> {
onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null);
});
return resultList;
}
@GetMapping("/branches")
@PreAuthorize("hasAuthority('TENANT_ADMIN')")
public Set<String> getAllowedBranches() throws ThingsboardException {
return versionControlService.getAllowedBranches(getTenantId());
}

184
application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java
View File

@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.SerializationFeature;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
@ -46,6 +47,8 @@ import org.thingsboard.server.utils.git.Repository;
import org.thingsboard.server.utils.git.data.Commit;
import java.io.File;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@ -54,7 +57,9 @@ import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
@Service
@ -72,8 +77,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private static final String SETTINGS_KEY = "vc";
private Repository repository;
private final ReentrantLock fetchLock = new ReentrantLock();
private final Lock writeLock = new ReentrantLock();
private final Lock fetchLock = new ReentrantLock();
private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock();
@AfterStartUp
public void init() throws Exception {
@ -89,17 +94,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000)
public void fetch() throws Exception {
private void fetch() throws Exception {
if (repository == null) return;
if (fetchLock.tryLock()) {
try {
log.info("Fetching remote repository");
repository.fetch();
} finally {
fetchLock.unlock();
}
}
tryFetch();
}
@ -118,20 +115,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.exportOutboundRelations(false)
.build();
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = entitiesIds.stream()
.map(entityId -> {
return exportImportService.exportEntity(tenantId, entityId, exportSettings);
})
.map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings))
.collect(Collectors.toList());
if (fetchLock.tryLock()) {
try {
repository.fetch();
} finally {
fetchLock.unlock();
}
}
tryFetch();
writeLock.lock();
repositoryLock.writeLock().lock();
try {
if (repository.listBranches().contains(branch)) {
repository.checkout(branch);
@ -148,9 +137,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId);
repository.push();
return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName());
return toVersion(commit);
} finally {
writeLock.unlock();
repositoryLock.writeLock().unlock();
}
}
@ -158,38 +147,72 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override
public List<EntityVersion> listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception {
checkRepository();
checkBranch(tenantId, branch);
return repository.listCommits(branch, getRelativePathForEntity(entityId), limit).stream()
.map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()))
.collect(Collectors.toList());
return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit);
}
@Override
public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception {
checkRepository();
checkBranch(tenantId, branch);
return listVersions(tenantId, getRelativePathForEntityType(entityType), limit);
}
return repository.listCommits(branch, getRelativePathForEntityType(entityType), limit).stream()
.map(commit -> new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()))
.collect(Collectors.toList());
@Override
public List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception {
return listVersions(tenantId, branch, null, limit);
}
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception {
repositoryLock.readLock().lock();
try {
checkRepository();
checkBranch(tenantId, branch);
return repository.listCommits(branch, path, limit).stream()
.map(this::toVersion)
.collect(Collectors.toList());
} finally {
repositoryLock.readLock().unlock();
}
}
@Override
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception {
checkRepository();
// FIXME [viacheslav]: validate access
public List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
repositoryLock.readLock().lock();
try {
if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream()
.noneMatch(version -> version.getId().equals(versionId))) {
throw new IllegalArgumentException("Unknown version");
}
return repository.listFilesAtCommit(versionId);
} finally {
repositoryLock.readLock().unlock();
}
}
String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId);
return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {});
@Override
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception {
repositoryLock.readLock().lock();
try {
if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream()
.noneMatch(version -> version.getId().equals(versionId))) {
throw new IllegalArgumentException("Unknown version");
}
String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId);
return parseEntityData(entityDataJson);
} finally {
repositoryLock.readLock().unlock();
}
}
@Override
public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception {
EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, versionId);
public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception {
EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, branch, versionId);
return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder()
.importInboundRelations(false)
.importOutboundRelations(false)
@ -197,6 +220,47 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.build());
}
@Override
public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception {
repositoryLock.readLock().lock();
try {
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream()
.map(entityDataFilePath -> {
String entityDataJson;
try {
entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
return parseEntityData(entityDataJson);
})
.collect(Collectors.toList());
return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder()
.importInboundRelations(false)
.importOutboundRelations(false)
.updateReferencesToOtherEntities(true)
.build());
} finally {
repositoryLock.readLock().unlock();
}
}
private void tryFetch() throws GitAPIException {
repositoryLock.readLock().lock();
try {
if (fetchLock.tryLock()) {
try {
log.info("Fetching remote repository");
repository.fetch();
} finally {
fetchLock.unlock();
}
}
} finally {
repositoryLock.readLock().unlock();
}
}
private String getRelativePathForEntity(EntityId entityId) {
@ -210,6 +274,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private void checkBranch(TenantId tenantId, String branch) {
// TODO [viacheslav]: all branches are available by default?
if (!getAllowedBranches(tenantId).contains(branch)) {
throw new IllegalArgumentException("Tenant does not have access to this branch");
}
@ -222,9 +287,24 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
.orElse(Collections.emptySet());
}
private EntityVersion toVersion(Commit commit) {
return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName());
}
private <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> parseEntityData(String entityDataJson) {
return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {});
}
@Override
public void saveSettings(EntitiesVersionControlSettings settings) throws Exception {
this.repository = initRepository(settings.getGitSettings());
repositoryLock.writeLock().lock();
try {
this.repository = initRepository(settings.getGitSettings());
} finally {
repositoryLock.writeLock().unlock();
}
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY))
.orElseGet(() -> {
@ -244,7 +324,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
private void checkRepository() {
if (repository == null) {
throw new IllegalStateException("Repository is not initialized");
@ -263,12 +342,17 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
public void resetRepository() throws Exception {
if (this.repository != null) {
FileUtils.deleteDirectory(new File(repository.getDirectory()));
this.repository = null;
repositoryLock.writeLock().lock();
try {
if (this.repository != null) {
FileUtils.deleteDirectory(new File(repository.getDirectory()));
this.repository = null;
}
EntitiesVersionControlSettings settings = getSettings();
this.repository = initRepository(settings.getGitSettings());
} finally {
repositoryLock.writeLock().unlock();
}
EntitiesVersionControlSettings settings = getSettings();
this.repository = initRepository(settings.getGitSettings());
}
}

11
application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java
View File

@ -37,10 +37,17 @@ public interface EntitiesVersionControlService {
List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception;
List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception;
<E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String versionId) throws Exception;
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String versionId) throws Exception;
List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
<E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception;
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception;
List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception;
void saveSettings(EntitiesVersionControlSettings settings) throws Exception;

8
application/src/main/java/org/thingsboard/server/utils/git/Repository.java
View File

@ -113,13 +113,13 @@ public class Repository {
}
public List<String> listFilesAtCommit(Commit commit) throws IOException {
return listFilesAtCommit(commit, null);
public List<String> listFilesAtCommit(String commitId) throws IOException {
return listFilesAtCommit(commitId, null);
}
public List<String> listFilesAtCommit(Commit commit, String path) throws IOException {
public List<String> listFilesAtCommit(String commitId, String path) throws IOException {
List<String> files = new ArrayList<>();
RevCommit revCommit = resolveCommit(commit.getId());
RevCommit revCommit = resolveCommit(commitId);
try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) {
treeWalk.reset(revCommit.getTree().getId());
if (StringUtils.isNotEmpty(path)) {

Write Preview
Loading…
Cancel
Save