From 073875f406ed4437ec844a3a29a116effba5be44 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Thu, 28 Apr 2022 13:26:11 +0300 Subject: [PATCH] Entities VC improvements and refactoring --- .../EntitiesVersionControlController.java | 272 ++++++------ .../DefaultEntitiesVersionControlService.java | 395 ++++++++++++++++++ .../vc/EntitiesVersionControlService.java | 64 +++ .../data/EntitiesVersionControlSettings.java} | 20 +- .../sync/{vcs => vc}/data/EntityVersion.java | 5 +- .../data/EntityVersionLoadResult.java} | 21 +- .../vc/data/EntityVersionLoadSettings.java} | 9 +- .../vc/data/EntityVersionSaveSettings.java} | 6 +- .../sync/vc/data/VersionedEntityInfo.java} | 11 +- .../DefaultEntitiesVersionControlService.java | 358 ---------------- .../vcs/EntitiesVersionControlService.java | 57 --- .../Repository.java => GitRepository.java} | 79 ++-- 12 files changed, 665 insertions(+), 632 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/GitSettings.java => vc/data/EntitiesVersionControlSettings.java} (75%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs => vc}/data/EntityVersion.java (90%) rename application/src/main/java/org/thingsboard/server/service/sync/{vcs/data/EntitiesVersionControlSettings.java => vc/data/EntityVersionLoadResult.java} (57%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Commit.java => service/sync/vc/data/EntityVersionLoadSettings.java} (78%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Branch.java => service/sync/vc/data/EntityVersionSaveSettings.java} (83%) rename application/src/main/java/org/thingsboard/server/{utils/git/data/Diff.java => service/sync/vc/data/VersionedEntityInfo.java} (74%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java delete mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java rename application/src/main/java/org/thingsboard/server/utils/{git/Repository.java => GitRepository.java} (82%) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 798556998c..c39442341e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -16,159 +16,139 @@ package org.thingsboard.server.controller; import lombok.RequiredArgsConstructor; -import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.EntityIdFactory; -import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.DefaultEntitiesVersionControlService; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.Arrays; -import java.util.List; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; +import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; @RestController @RequestMapping("/api/entities/vc") @RequiredArgsConstructor public class EntitiesVersionControlController extends BaseController { - private final DefaultEntitiesVersionControlService versionControlService; - - - - @PostMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID id, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); - return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); - } - - @PostMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, - @RequestParam UUID[] ids, - @RequestParam String branch, - @RequestBody String versionName) throws Exception { - List entitiesIds = Arrays.stream(ids) - .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) - .collect(Collectors.toList()); - return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); - } - - - - @GetMapping("/version/{entityType}/{entityId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityVersions(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version/{entityType}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listEntityTypeVersions(@PathVariable EntityType entityType, - @RequestParam String branch) throws Exception { - return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); - } - - @GetMapping("/version") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listVersions(@RequestParam String branch) throws Exception { - return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); - } - - - - @GetMapping("/files/version/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List listFilesAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); - } - - - - @GetMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); - } - - @PostMapping("/entity/{entityType}/{entityId}/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, - @PathVariable("entityId") UUID entityUuid, - @RequestParam String branch, - @PathVariable String versionId) throws Exception { - EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); - onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - return result; - } - - @PostMapping("/entity/{versionId}") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public List>> loadAllAtVersion(@RequestParam String branch, - @PathVariable String versionId) throws Exception { - SecurityUser user = getCurrentUser(); - List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); - resultList.forEach(result -> { - onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); - }); - return resultList; - } - - - - @GetMapping("/branches") - @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public Set getAllowedBranches() throws ThingsboardException { - return versionControlService.getAllowedBranches(getTenantId()); - } - - - @PostMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { - versionControlService.saveSettings(settings); - } - - @GetMapping("/settings") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public EntitiesVersionControlSettings getSettings() { - return versionControlService.getSettings(); - } - - - - @PostMapping("/repository/reset") - @PreAuthorize("hasAuthority('SYS_ADMIN')") - public void resetLocalRepository() throws Exception { - versionControlService.resetRepository(); - } + private final EntitiesVersionControlService versionControlService; + + + // search request - export request with settings + +// +// @PostMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID id, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, id); +// return versionControlService.saveEntityVersion(getTenantId(), entityId, branch, versionName); +// } +// +// @PostMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityVersion saveEntitiesVersion(@PathVariable EntityType entityType, +// @RequestParam UUID[] ids, +// @RequestParam String branch, +// @RequestBody String versionName) throws Exception { +// List entitiesIds = Arrays.stream(ids) +// .map(id -> EntityIdFactory.getByTypeAndUuid(entityType, id)) +// .collect(Collectors.toList()); +// return versionControlService.saveEntitiesVersion(getTenantId(), entitiesIds, branch, versionName); +// } +// +// +// +// @GetMapping("/version/{entityType}/{entityId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityVersions(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.listEntityVersions(getTenantId(), entityId, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version/{entityType}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listEntityTypeVersions(@PathVariable EntityType entityType, +// @RequestParam String branch) throws Exception { +// return versionControlService.listEntityTypeVersions(getTenantId(), entityType, branch, Integer.MAX_VALUE); +// } +// +// @GetMapping("/version") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listVersions(@RequestParam String branch) throws Exception { +// return versionControlService.listVersions(getTenantId(), branch, Integer.MAX_VALUE); +// } +// +// +// +// @GetMapping("/files/version/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List listFilesAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// return versionControlService.listFilesAtVersion(getTenantId(), branch, versionId); +// } +// +// +// +// @GetMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityExportData> getEntityAtVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// return versionControlService.getEntityAtVersion(getTenantId(), entityId, branch, versionId); +// } +// +// @PostMapping("/entity/{entityType}/{entityId}/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public EntityImportResult> loadEntityVersion(@PathVariable EntityType entityType, +// @PathVariable("entityId") UUID entityUuid, +// @RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); +// EntityImportResult> result = versionControlService.loadEntityVersion(getTenantId(), entityId, branch, versionId); +// onEntityUpdatedOrCreated(getCurrentUser(), result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// return result; +// } +// +// @PostMapping("/entity/{versionId}") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public List>> loadAllAtVersion(@RequestParam String branch, +// @PathVariable String versionId) throws Exception { +// SecurityUser user = getCurrentUser(); +// List>> resultList = versionControlService.loadAllAtVersion(user.getTenantId(), branch, versionId); +// resultList.forEach(result -> { +// onEntityUpdatedOrCreated(user, result.getSavedEntity(), result.getOldEntity(), result.getOldEntity() == null); +// }); +// return resultList; +// } +// +// +// +// @GetMapping("/branches") +// @PreAuthorize("hasAuthority('TENANT_ADMIN')") +// public Set getAllowedBranches() throws ThingsboardException { +// return versionControlService.getAllowedBranches(getTenantId()); +// } +// +// +// @PostMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void saveSettings(@RequestBody EntitiesVersionControlSettings settings) throws Exception { +// versionControlService.saveSettings(settings); +// } +// +// @GetMapping("/settings") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public EntitiesVersionControlSettings getSettings() { +// return versionControlService.getSettings(); +// } +// +// +// +// @PostMapping("/repository/reset") +// @PreAuthorize("hasAuthority('SYS_ADMIN')") +// public void resetLocalRepository() throws Exception { +// versionControlService.resetRepository(); +// } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java new file mode 100644 index 0000000000..a298d64ccc --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -0,0 +1,395 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; +import org.thingsboard.server.queue.util.AfterStartUp; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.EntitiesExportImportService; +import org.thingsboard.server.service.sync.exporting.data.EntityExportData; +import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; +import org.thingsboard.server.service.sync.importing.data.EntityImportResult; +import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; +import org.thingsboard.server.utils.GitRepository; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.stream.Collectors; + +@Service +@TbCoreComponent +@RequiredArgsConstructor +@Slf4j +public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + + private final EntitiesExportImportService exportImportService; + + private GitRepository repository; + private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); + + private ScheduledExecutorService fetchExecutor; + private ScheduledFuture fetchTask; + + private final AdminSettingsService adminSettingsService; + private static final String SETTINGS_KEY = "vc"; + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + + + @AfterStartUp + public void init() { + EntitiesVersionControlSettings settings = getSettings(); + if (settings != null) { + try { + initRepository(settings); + } catch (Exception e) { + log.debug("Failed to init repository", e); + } + } + + int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); + fetchExecutor = Executors.newSingleThreadScheduledExecutor(); + fetchTask = scheduleFetch(fetchPeriod); + } + + + @Override + public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); + } + + @Override + public EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + checkRepository(); + checkBranch(user.getTenantId(), branch); + + List> entityDataList = new ArrayList<>(); + EntityExportSettings exportSettings = EntityExportSettings.builder() + .exportRelations(settings.isSaveRelations()) + .build(); + for (EntityId entityId : entitiesIds) { + EntityExportData> entityData = exportImportService.exportEntity(user, entityId, exportSettings); + entityDataList.add(entityData); + } + + fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout(branch); + repository.merge(branch); + } else { + repository.createAndCheckoutOrphanBranch(branch); + } + + for (EntityExportData entityData : entityDataList) { + String entityDataJson = jsonWriter.writeValueAsString(entityData); + FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); + } + + GitRepository.Commit commit = repository.commit(versionName, "."); + repository.push(); + return toVersion(commit); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @Override + public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { + return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + } + + @Override + public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch) throws Exception { + return listVersions(tenantId, branch, null); + } + + private List listVersions(TenantId tenantId, String branch, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + } + + @Override + public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + repositoryLock.readLock().lock(); + try { + checkRepository(); + checkBranch(tenantId, branch); + checkVersion(tenantId, branch, versionId, path); + + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + EntityExportData entityData = getEntityDataAtVersion(entityId, versionId); + + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + info.setEntityName(entityData.getEntity().getName()); + return info; + }) + .collect(Collectors.toList()); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + @Override + public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); + } + + @Override + public List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); + } + + @Override + public List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { + return loadAtVersion(user, branch, versionId, null, settings); + } + + private List loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { + List> entityDataList = new ArrayList<>(); + repositoryLock.readLock().lock(); + try { + for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { + EntityExportData entityData = getEntityDataAtVersion(info.getExternalId(), versionId); + entityDataList.add(entityData); + } + } finally { + repositoryLock.readLock().unlock(); + } + + EntityImportSettings importSettings = EntityImportSettings.builder() + .updateRelations(settings.isLoadRelations()) + .findExistingByName(settings.isFindExistingEntityByName()) + .build(); + List> importResults = exportImportService.importEntities(user, entityDataList, importSettings); + + return importResults.stream() + .map(importResult -> EntityVersionLoadResult.builder() + .previousEntityVersion(importResult.getOldEntity()) + .newEntityVersion(importResult.getSavedEntity()) + .entityType(importResult.getEntityType()) + .build()) + .collect(Collectors.toList()); + } + + @SneakyThrows + private EntityExportData getEntityDataAtVersion(EntityId externalId, String versionId) { + repositoryLock.readLock().lock(); + try { + String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } finally { + repositoryLock.readLock().unlock(); + } + } + + + private void fetch() throws GitAPIException { + repositoryLock.writeLock().lock(); + try { + repository.fetch(); + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private ScheduledFuture scheduleFetch(int fetchPeriod) { + return fetchExecutor.scheduleWithFixedDelay(() -> { + if (repository == null) return; + try { + fetch(); + } catch (Exception e) { + log.error("Failed to fetch remote repository", e); + } + }, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); + } + + + private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { + throw new IllegalArgumentException("Version not found"); + } + } + + @Override + public List listAllowedBranches(TenantId tenantId) { + return Optional.ofNullable(getSettings()) + .flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) + .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) + .orElse(Collections.emptyList()); + } + + private void checkBranch(TenantId tenantId, String branch) { + if (!listAllowedBranches(tenantId).contains(branch)) { + throw new IllegalArgumentException("Tenant does not have access to the branch"); + } + } + + + private void checkRepository() { + if (repository == null) { + throw new IllegalStateException("Repository is not initialized"); + } + } + + private void initRepository(EntitiesVersionControlSettings settings) throws Exception { + repositoryLock.writeLock().lock(); + try { + if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { + this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); + } else { + Files.createDirectories(Path.of(settings.getRepositoryDirectory())); + this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), + settings.getUsername(), settings.getPassword()); + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + private void clearRepository() throws IOException { + repositoryLock.writeLock().lock(); + try { + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repository = null; + } + } finally { + repositoryLock.writeLock().unlock(); + } + } + + + @SneakyThrows + @Override + public void saveSettings(EntitiesVersionControlSettings settings) { + AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .orElseGet(() -> { + AdminSettings newAdminSettings = new AdminSettings(); + newAdminSettings.setKey(SETTINGS_KEY); + return newAdminSettings; + }); + adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); + + repositoryLock.writeLock().lock(); + try { + clearRepository(); + initRepository(settings); + } finally { + repositoryLock.writeLock().unlock(); + } + + if (settings.getFetchPeriod() != 0) { + fetchTask.cancel(true); + fetchTask = scheduleFetch(settings.getFetchPeriod()); + } + } + + @Override + public EntitiesVersionControlSettings getSettings() { + return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) + .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) + .orElse(null); + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java new file mode 100644 index 0000000000..435c661b02 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; +import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; +import org.thingsboard.server.service.sync.vc.data.EntityVersion; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; +import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; +import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; + +import java.util.List; + +public interface EntitiesVersionControlService { + + EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + EntityVersion saveEntitiesVersion(SecurityUser user, List entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; + + + List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; + + List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; + + List listVersions(TenantId tenantId, String branch) throws Exception; + + + List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also + + List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; + + + EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + List loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; + + + List listAllowedBranches(TenantId tenantId); + + + void saveSettings(EntitiesVersionControlSettings settings); + + EntitiesVersionControlSettings getSettings(); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java index 0d97bed2d1..06e0a9c7aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/GitSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesVersionControlSettings.java @@ -13,20 +13,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; -import lombok.AllArgsConstructor; -import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; +import java.util.UUID; @Data -@AllArgsConstructor -@NoArgsConstructor -@Builder -public class GitSettings { - private String repositoryUri; +public class EntitiesVersionControlSettings { private String repositoryDirectory; + private String repositoryUri; private String username; private String password; + + private int fetchPeriod; + + private Map> tenantsAllowedBranches; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java index 42a2e2f555..3547f89e3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntityVersion.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java @@ -13,17 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @Data -@NoArgsConstructor @AllArgsConstructor +@NoArgsConstructor public class EntityVersion { private String id; private String name; - private String authorName; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java similarity index 57% rename from application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java index 6a46606845..4cf0ee2b0f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/data/EntitiesVersionControlSettings.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadResult.java @@ -13,16 +13,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vcs.data; +package org.thingsboard.server.service.sync.vc.data; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; - -import java.util.Map; -import java.util.Set; -import java.util.UUID; +import lombok.NoArgsConstructor; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; @Data -public class EntitiesVersionControlSettings { - private Map> allowedBranches; - private GitSettings gitSettings; +@AllArgsConstructor +@NoArgsConstructor +@Builder +public class EntityVersionLoadResult { + private ExportableEntity newEntityVersion; + private ExportableEntity previousEntityVersion; + private EntityType entityType; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java index 7567dba1c5..842f6ddd4e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Commit.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionLoadSettings.java @@ -13,13 +13,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Commit { - private final String id; - private final String message; - private final String authorName; +public class EntityVersionLoadSettings { + private boolean loadRelations; + private boolean findExistingEntityByName; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java index a458f9f5aa..1fc36463ba 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Branch.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersionSaveSettings.java @@ -13,11 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; @Data -public class Branch { - private final String shortName; +public class EntityVersionSaveSettings { + private boolean saveRelations; } diff --git a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java rename to application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java index 7572a003d2..97bb6ada12 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/data/Diff.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java @@ -13,13 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git.data; +package org.thingsboard.server.service.sync.vc.data; import lombok.Data; +import org.thingsboard.server.common.data.id.EntityId; @Data -public class Diff { - private final String type; - private final String oldPath; - private final String newPath; +public class VersionedEntityInfo { + private EntityId externalId; + private String entityName; + // etc.. } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java deleted file mode 100644 index 32da1309fa..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/DefaultEntitiesVersionControlService.java +++ /dev/null @@ -1,358 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantService; -import org.thingsboard.server.queue.util.AfterStartUp; -import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exporting.EntityExportSettings; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.importing.EntityImportSettings; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; -import org.thingsboard.server.service.sync.vcs.data.GitSettings; -import org.thingsboard.server.utils.git.Repository; -import org.thingsboard.server.utils.git.data.Commit; - -import java.io.File; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.stream.Collectors; - -@Service -@TbCoreComponent -@RequiredArgsConstructor -@Slf4j -public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { - // TODO [viacheslav]: start up only on one of the cores - - private final TenantService tenantService; - private final EntitiesExportImportService exportImportService; - private final AdminSettingsService adminSettingsService; - - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - private static final String SETTINGS_KEY = "vc"; - - private Repository repository; - private final Lock fetchLock = new ReentrantLock(); - private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); - - @AfterStartUp - public void init() throws Exception { - try { - EntitiesVersionControlSettings settings = getSettings(); - if (settings != null && settings.getGitSettings() != null) { - this.repository = initRepository(settings.getGitSettings()); - } - } catch (Exception e) { - log.error("Failed to initialize entities version control service", e); - } - } - - - @Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) - private void fetch() throws Exception { - if (repository == null) return; - tryFetch(); - } - - - @Override - public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { - return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); - } - - @Override - public EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception { - checkRepository(); - checkBranch(tenantId, branch); - - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportInboundRelations(false) - .exportOutboundRelations(false) - .build(); - List>> entityDataList = entitiesIds.stream() - .map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) - .collect(Collectors.toList()); - - tryFetch(); - - repositoryLock.writeLock().lock(); - try { - if (repository.listBranches().contains(branch)) { - repository.checkout(branch); - repository.merge(branch); - } else { - repository.createAndCheckoutOrphanBranch(branch); - } - - for (EntityExportData> entityData : entityDataList) { - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), - entityDataJson, StandardCharsets.UTF_8); - } - - Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); - repository.push(); - return toVersion(commit); - } finally { - repositoryLock.writeLock().unlock(); - } - } - - - - @Override - public List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); - } - - @Override - public List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { - return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); - } - - @Override - public List listVersions(TenantId tenantId, String branch, int limit) throws Exception { - return listVersions(tenantId, branch, null, limit); - } - - private List listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { - repositoryLock.readLock().lock(); - try { - checkRepository(); - checkBranch(tenantId, branch); - - return repository.listCommits(branch, path, limit).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - return repository.listFilesAtCommit(versionId); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - - @Override - public , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() - .noneMatch(version -> version.getId().equals(versionId))) { - throw new IllegalArgumentException("Unknown version"); - } - - String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); - return parseEntityData(entityDataJson); - } finally { - repositoryLock.readLock().unlock(); - } - } - - - @Override - public , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { - EntityExportData entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); - return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } - - @Override - public List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - repositoryLock.readLock().lock(); - try { - List>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() - .map(entityDataFilePath -> { - String entityDataJson; - try { - entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return parseEntityData(entityDataJson); - }) - .collect(Collectors.toList()); - - return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() - .importInboundRelations(false) - .importOutboundRelations(false) - .updateReferencesToOtherEntities(true) - .build()); - } finally { - repositoryLock.readLock().unlock(); - } - } - - private void tryFetch() throws GitAPIException { - repositoryLock.readLock().lock(); - try { - if (fetchLock.tryLock()) { - try { - log.info("Fetching remote repository"); - repository.fetch(); - } finally { - fetchLock.unlock(); - } - } - } finally { - repositoryLock.readLock().unlock(); - } - } - - - private String getRelativePathForEntity(EntityId entityId) { - return getRelativePathForEntityType(entityId.getEntityType()) - + "/" + entityId.getId() + ".json"; - } - - private String getRelativePathForEntityType(EntityType entityType) { - return entityType.name().toLowerCase(); - } - - - private void checkBranch(TenantId tenantId, String branch) { - // TODO [viacheslav]: all branches are available by default? - if (!getAllowedBranches(tenantId).contains(branch)) { - throw new IllegalArgumentException("Tenant does not have access to this branch"); - } - } - - public Set getAllowedBranches(TenantId tenantId) { - return Optional.ofNullable(getSettings()) - .flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) - .flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) - .orElse(Collections.emptySet()); - } - - private EntityVersion toVersion(Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); - } - - private , I extends EntityId> EntityExportData parseEntityData(String entityDataJson) { - return JacksonUtil.fromString(entityDataJson, new TypeReference>() {}); - } - - - - @Override - public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { - repositoryLock.writeLock().lock(); - try { - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - - AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .orElseGet(() -> { - AdminSettings newSettings = new AdminSettings(); - newSettings.setKey(SETTINGS_KEY); - return newSettings; - }); - adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); - } - - @Override - public EntitiesVersionControlSettings getSettings() { - return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) - .map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) - .orElse(null); - } - - - private void checkRepository() { - if (repository == null) { - throw new IllegalStateException("Repository is not initialized"); - } - } - - private static Repository initRepository(GitSettings gitSettings) throws Exception { - if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { - return Repository.open(gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } else { - Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); - return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), - gitSettings.getUsername(), gitSettings.getPassword()); - } - } - - public void resetRepository() throws Exception { - repositoryLock.writeLock().lock(); - try { - if (this.repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - this.repository = null; - } - EntitiesVersionControlSettings settings = getSettings(); - this.repository = initRepository(settings.getGitSettings()); - } finally { - repositoryLock.writeLock().unlock(); - } - } - -} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java deleted file mode 100644 index e05ad17826..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/sync/vcs/EntitiesVersionControlService.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.sync.vcs; - -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.ExportableEntity; -import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.service.sync.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.importing.EntityImportResult; -import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; -import org.thingsboard.server.service.sync.vcs.data.EntityVersion; - -import java.util.List; - -public interface EntitiesVersionControlService { - - EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; - - EntityVersion saveEntitiesVersion(TenantId tenantId, List entitiesIds, String branch, String versionName) throws Exception; - - - List listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; - - List listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; - - List listVersions(TenantId tenantId, String branch, int limit) throws Exception; - - - List listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - , I extends EntityId> EntityExportData getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - , I extends EntityId> EntityImportResult loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; - - List>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; - - - void saveSettings(EntitiesVersionControlSettings settings) throws Exception; - - EntitiesVersionControlSettings getSettings(); - -} diff --git a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java similarity index 82% rename from application/src/main/java/org/thingsboard/server/utils/git/Repository.java rename to application/src/main/java/org/thingsboard/server/utils/GitRepository.java index 23eb7d7db4..205bbca50e 100644 --- a/application/src/main/java/org/thingsboard/server/utils/git/Repository.java +++ b/application/src/main/java/org/thingsboard/server/utils/GitRepository.java @@ -13,9 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils.git; +package org.thingsboard.server.utils; import com.google.common.collect.Streams; +import lombok.Data; import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.eclipse.jgit.api.Git; @@ -32,19 +33,21 @@ import org.eclipse.jgit.lib.ObjectReader; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.filter.RevFilter; import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.URIish; import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; -import org.thingsboard.server.utils.git.data.Commit; +import java.io.File; import java.io.IOException; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; -public class Repository { +public class GitRepository { private final Git git; private final CredentialsProvider credentialsProvider; @@ -52,13 +55,13 @@ public class Repository { @Getter private final String directory; - private Repository(Git git, CredentialsProvider credentialsProvider, String directory) { + private GitRepository(Git git, CredentialsProvider credentialsProvider, String directory) { this.git = git; this.credentialsProvider = credentialsProvider; this.directory = directory; } - public static Repository clone(String uri, String directory, + public static GitRepository clone(String uri, String directory, String username, String password) throws GitAPIException { CredentialsProvider credentialsProvider = newCredentialsProvider(username, password); Git git = Git.cloneRepository() @@ -67,12 +70,12 @@ public class Repository { .setNoCheckout(true) .setCredentialsProvider(credentialsProvider) .call(); - return new Repository(git, credentialsProvider, directory); + return new GitRepository(git, credentialsProvider, directory); } - public static Repository open(String directory, String username, String password) throws IOException { + public static GitRepository open(String directory, String username, String password) throws IOException { Git git = Git.open(new java.io.File(directory)); - return new Repository(git, newCredentialsProvider(username, password), directory); + return new GitRepository(git, newCredentialsProvider(username, password), directory); } @@ -81,6 +84,20 @@ public class Repository { .setRemoveDeletedRefs(true)); } + public void checkout(String branch) throws GitAPIException { + execute(git.checkout() + .setName(branch)); + } + + public void merge(String branch) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); + if (branchId == null) { + throw new IllegalArgumentException("Branch not found"); + } + execute(git.merge() + .include(branchId)); + } + public List listBranches() throws GitAPIException { return execute(git.branchList() @@ -92,12 +109,12 @@ public class Repository { } - public List listCommits(String branchName, int limit) throws IOException, GitAPIException { - return listCommits(branchName, null, limit); + public List listCommits(String branch, int limit) throws IOException, GitAPIException { + return listCommits(branch, null, limit); } - public List listCommits(String branchName, String path, int limit) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); + public List listCommits(String branch, String path, int limit) throws IOException, GitAPIException { + ObjectId branchId = resolve("origin/" + branch); if (branchId == null) { throw new IllegalArgumentException("Branch not found"); } @@ -150,20 +167,6 @@ public class Repository { } - public void checkout(String branchName) throws GitAPIException { - execute(git.checkout() - .setName(branchName)); - } - - public void merge(String branchName) throws IOException, GitAPIException { - ObjectId branchId = resolve("origin/" + branchName); - if (branchId == null) { - throw new IllegalArgumentException("Branch not found"); - } - execute(git.merge() - .include(branchId)); - } - public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) @@ -177,15 +180,11 @@ public class Repository { execute(git.clean()); } - public void clean() throws GitAPIException { - execute(git.clean().setCleanDirectories(true)); - } - public Commit commit(String message, String filePattern, String author) throws GitAPIException { - execute(git.add().addFilepattern(filePattern)); + public Commit commit(String message, String filesPattern) throws GitAPIException { + execute(git.add().addFilepattern(filesPattern)); RevCommit revCommit = execute(git.commit() - .setMessage(message) - .setAuthor(author, author)); + .setMessage(message)); // TODO [viacheslav]: set configurable author for commit return toCommit(revCommit); } @@ -239,12 +238,8 @@ public class Repository { } private , T> T execute(C command) throws GitAPIException { - if (command instanceof TransportCommand) { + if (command instanceof TransportCommand && credentialsProvider != null) { ((TransportCommand) command).setCredentialsProvider(credentialsProvider); -// SshSessionFactory sshSessionFactory = SshSessionFactory.getInstance(); -// transportCommand.setTransportConfigCallback(transport -> { -// ((SshTransport) transport).setSshSessionFactory(sshSessionFactory); -// }); } return command.call(); } @@ -253,4 +248,12 @@ public class Repository { return new UsernamePasswordCredentialsProvider(username, password); } + + @Data + public static class Commit { + private final String id; + private final String message; + private final String authorName; + } + }