diff --git a/application/pom.xml b/application/pom.xml index 2ff2204b5a..37d782b3b0 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -345,10 +345,6 @@ Java-WebSocket test - - org.eclipse.jgit - org.eclipse.jgit - diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 2ab75c565d..2794db90d4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -172,7 +172,7 @@ public class EntitiesVersionControlController extends BaseController { @PathVariable EntityType entityType, @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index c22eb6a5a5..e222d027e7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -75,7 +75,6 @@ import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersion import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; -import org.thingsboard.server.utils.GitRepository; import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.io.File; @@ -105,85 +104,31 @@ import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final GitVersionControlService gitService; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AttributesService attributesService; private final EntityService entityService; - private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; - // TODO [viacheslav]: concurrency - private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}/repositories") - private String repositoriesFolder; - - private static final String SETTINGS_KEY = "vc"; - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getSettings(tenantId); - if (settings != null) { - try { - initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, 5, 5, TimeUnit.SECONDS); - } - + public static final String SETTINGS_KEY = "vc"; @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout("origin/" + request.getBranch(), false); - try { - repository.checkout(request.getBranch(), true); - } catch (RefAlreadyExistsException e) { - repository.checkout(request.getBranch(), false); - } - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - try { - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } catch (JGitInternalException e) { - if (!e.getMessage().contains("NO_CHANGE")) { - throw e; - } - } - } + + var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); + saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } + gitService.deleteAll(commit, entityType); } if (config.isAllEntities()) { @@ -203,7 +148,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, 100, data -> { EntityId entityId = data.getEntityId(); try { - saveEntityData(user, repository, entityId, config); + saveEntityData(user, commit, entityId, config); } catch (Exception e) { throw new RuntimeException(e); } @@ -211,7 +156,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else { for (UUID entityId : config.getEntityIds()) { try { - saveEntityData(user, repository, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); } catch (Exception e) { throw new RuntimeException(e); } @@ -223,90 +168,50 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); - - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); - - result.setVersion(toVersion(commit)); - return result; + return gitService.push(commit); } - private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + gitService.addToCommit(commit, entityData); } @Override public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + return gitService.listVersions(tenantId, branch, externalId); } @Override public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + return gitService.listVersions(tenantId, branch, entityType); } @Override public List listVersions(TenantId tenantId, String branch) throws Exception { - return listVersions(tenantId, branch, null); - } - - private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); + return gitService.listVersions(tenantId, branch); } - @Override - public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, null); + return gitService.listEntitiesAtVersion(tenantId, branch, versionId); } - private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - return info; - }) - .collect(Collectors.toList()); - } - - @Override public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); + EntityImportResult result = loadEntity(user, request, versionLoadRequest.getConfig(), versionLoadRequest.getExternalEntityId()); result.getSaveReferencesCallback().run(); return result; } catch (Exception e) { @@ -340,11 +245,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont Set remoteEntities; try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() + remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType).stream() .map(VersionedEntityInfo::getExternalId) .collect(Collectors.toSet()); for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); + EntityImportResult importResult = loadEntity(user, request, config, externalEntityId); if (importResult.getOldEntity() == null) created.incrementAndGet(); else updated.incrementAndGet(); @@ -402,10 +307,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); - + private EntityImportResult loadEntity(SecurityUser user, VersionLoadRequest request, VersionLoadConfig config, EntityId entityId) throws Exception { + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), entityId); return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) .findExistingByName(config.isFindExistingEntityByName()) @@ -415,43 +318,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @Override public List listBranches(TenantId tenantId) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listBranches(); - } - - - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - - private GitRepository checkRepository(TenantId tenantId) { - return Optional.ofNullable(repositories.get(tenantId)) - .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); - } - - private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { - Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - GitRepository repository; - if (Files.exists(repositoryDirectory)) { - FileUtils.forceDelete(repositoryDirectory.toFile()); - } - - Files.createDirectories(repositoryDirectory); - repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); - repositories.put(tenantId, repository); - } - - private void clearRepository(TenantId tenantId) throws IOException { - GitRepository repository = repositories.get(tenantId); - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repositories.remove(tenantId); - } + return gitService.listBranches(tenantId); } - @SneakyThrows @Override public void saveSettings(TenantId tenantId, EntitiesVersionControlSettings settings) { @@ -459,41 +328,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont new BaseAttributeKvEntry(System.currentTimeMillis(), new JsonDataEntry(SETTINGS_KEY, JacksonUtil.toString(settings))) )).get(); - initRepository(tenantId, settings); + gitService.initRepository(tenantId, settings); } - @SneakyThrows @Override public EntitiesVersionControlSettings getSettings(TenantId tenantId) { - return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, SETTINGS_KEY).get() - .flatMap(KvEntry::getJsonValue) - .map(json -> { - try { - return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); - } catch (IllegalArgumentException e) { - return null; - } - }) - .orElse(null); - } - - - private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; + return gitService.getSettings(tenantId); } - - private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); - String entityId = StringUtils.substringBetween(path, "/", ".json"); - return EntityIdFactory.getByTypeAndUuid(entityType, entityId); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index e025f80fed..f0d4169b1b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -41,7 +41,7 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java new file mode 100644 index 0000000000..b9a5a9260b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -0,0 +1,258 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.io.IOException; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; + +@Slf4j +@RequiredArgsConstructor +@Service +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +public class LocalGitVersionControlService implements GitVersionControlService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final GitRepositoryService gitRepositoryService; + private final TenantDao tenantDao; + private final AttributesService attributesService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); + private final Map pendingCommitMap = new HashMap<>(); + + @AfterStartUp + public void init() { + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = getSettings(tenantId); + if (settings != null) { + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } + } + }); + } + + @Override + @SneakyThrows + public EntitiesVersionControlSettings getSettings(TenantId tenantId) { + return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, DefaultEntitiesVersionControlService.SETTINGS_KEY).get() + .flatMap(KvEntry::getJsonValue) + .map(json -> { + try { + return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); + } catch (IllegalArgumentException e) { + return null; + } + }) + .orElse(null); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + + @Override + public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + var pendingCommit = new PendingCommit(tenantId, request); + PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); + if (old != null) { + gitRepositoryService.abort(old); + } + gitRepositoryService.prepareCommit(pendingCommit); + return pendingCommit; + } finally { + lock.unlock(); + } + } + + @Override + public void deleteAll(PendingCommit commit, EntityType entityType) { + doInsideLock(commit, c -> { + try { + gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public void addToCommit(PendingCommit commit, EntityExportData> entityData) { + doInsideLock(commit, c -> { + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString()), entityDataJson); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + return executeInsideLock(commit, gitRepositoryService::push); + } + + @Override + public List listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, branch, (String) null); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + try { + return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + @Override + public List listBranches(TenantId tenantId) { + return gitRepositoryService.listBranches(tenantId); + } + + @Override + public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { + try { + String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, + getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private List listVersions(TenantId tenantId, String branch, String path) { + try { + return gitRepositoryService.listVersions(tenantId, branch, path); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private void doInsideLock(PendingCommit commit, Consumer r) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + r.accept(commit); + } finally { + lock.unlock(); + } + } + + private T executeInsideLock(PendingCommit commit, Function c) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + return c.apply(commit); + } finally { + lock.unlock(); + } + } + + private void checkCommit(PendingCommit commit) { + PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); + if (existing == null || !existing.getTxId().equals(commit.getTxId())) { + throw new ConcurrentModificationException(); + } + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private Lock getRepoLock(TenantId tenantId) { + return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 03dbb49763..654929f500 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1115,6 +1115,11 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +vc: + git: + service: "${JS_VC_GIT_SERVICE:local}" # local/remote + repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + management: endpoints: web: diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 1b7194e029..f29b7db81a 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -40,6 +40,23 @@ org.springframework spring-core + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot-starter-web + provided + + + javax.annotation + javax.annotation-api + com.google.guava guava @@ -65,6 +82,10 @@ ch.qos.logback logback-classic + + org.eclipse.jgit + org.eclipse.jgit + org.springframework.boot spring-boot-starter-test diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java new file mode 100644 index 0000000000..d22fcf5d94 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -0,0 +1,239 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Slf4j +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +@Service +public class DefaultGitRepositoryService implements GitRepositoryService { + + @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + @Value("${vc.git.repos-poll-interval:60}") + private long reposPollInterval; + + private ScheduledExecutorService scheduler; + private final Map repositories = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + scheduler = Executors.newSingleThreadScheduledExecutor(); + scheduler.scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void prepareCommit(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + String branch = commit.getRequest().getBranch(); + try { + repository.fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout("origin/" + branch, false); + try { + repository.checkout(branch, true); + } catch (RefAlreadyExistsException e) { + repository.checkout(branch, false); + } + repository.merge(branch); + } else { // TODO [viacheslav]: rollback orphan branch on failure + try { + repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + } + } catch (IOException | GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); + } + + @Override + public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); + + GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); + repository.push(); + + result.setVersion(toVersion(gitCommit)); + return result; + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void abort(PendingCommit commit) { + //TODO: implement; + } + + @Override + public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); + } + + @Override + public List listBranches(TenantId tenantId) { + GitRepository repository = checkRepository(tenantId); + try { + return repository.listBranches(); + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } + + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); + } + + @Override + public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); + repositories.put(tenantId, repository); + } + + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); + } + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/utils/GitRepository.java rename to common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index ac8df955d6..afebe1f7fc 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java new file mode 100644 index 0000000000..2191f38930 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -0,0 +1,48 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.io.IOException; +import java.util.List; + +public interface GitRepositoryService { + + void prepareCommit(PendingCommit pendingCommit); + + List listVersions(TenantId tenantId, String branch, String path) throws Exception; + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; + + void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; + + VersionCreationResult push(PendingCommit commit); + + void abort(PendingCommit commit); + + List listBranches(TenantId tenantId); + + String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java new file mode 100644 index 0000000000..e67a8cca5f --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlService { + + @SneakyThrows + EntitiesVersionControlSettings getSettings(TenantId tenantId); + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); + + void addToCommit(PendingCommit commit, EntityExportData> entityData); + + void deleteAll(PendingCommit pendingCommit, EntityType entityType); + + VersionCreationResult push(PendingCommit commit); + + List listVersions(TenantId tenantId, String branch); + + List listVersions(TenantId tenantId, String branch, EntityType entityType); + + List listVersions(TenantId tenantId, String branch, EntityId entityId); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + List listBranches(TenantId tenantId); + + EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java new file mode 100644 index 0000000000..a30f7514a2 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -0,0 +1,36 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.UUID; + +@Data +public class PendingCommit { + + private final UUID txId; + private final TenantId tenantId; + private final VersionCreateRequest request; + + public PendingCommit(TenantId tenantId, VersionCreateRequest request) { + this.txId = UUID.randomUUID(); + this.tenantId = tenantId; + this.request = request; + } +}