Andrew Shvayka
4 years ago
committed by
GitHub
GitHub
68 changed files with 936 additions and 334 deletions
@ -0,0 +1,258 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2022 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.sync.vc; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
|
import com.fasterxml.jackson.databind.ObjectWriter; |
||||
|
import com.fasterxml.jackson.databind.SerializationFeature; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.SneakyThrows; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.server.common.data.DataConstants; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.ExportableEntity; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.kv.KvEntry; |
||||
|
import org.thingsboard.server.common.data.sync.ie.EntityExportData; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
||||
|
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; |
||||
|
import org.thingsboard.server.dao.DaoUtil; |
||||
|
import org.thingsboard.server.dao.attributes.AttributesService; |
||||
|
import org.thingsboard.server.dao.tenant.TenantDao; |
||||
|
import org.thingsboard.server.queue.util.AfterStartUp; |
||||
|
|
||||
|
import java.io.IOException; |
||||
|
import java.util.ConcurrentModificationException; |
||||
|
import java.util.HashMap; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.ConcurrentMap; |
||||
|
import java.util.concurrent.locks.Lock; |
||||
|
import java.util.concurrent.locks.ReentrantLock; |
||||
|
import java.util.function.Consumer; |
||||
|
import java.util.function.Function; |
||||
|
|
||||
|
@Slf4j |
||||
|
@RequiredArgsConstructor |
||||
|
@Service |
||||
|
@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) |
||||
|
public class LocalGitVersionControlService implements GitVersionControlService { |
||||
|
|
||||
|
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); |
||||
|
private final GitRepositoryService gitRepositoryService; |
||||
|
private final TenantDao tenantDao; |
||||
|
private final AttributesService attributesService; |
||||
|
private final ConcurrentMap<TenantId, Lock> tenantRepoLocks = new ConcurrentHashMap<>(); |
||||
|
private final Map<TenantId, PendingCommit> pendingCommitMap = new HashMap<>(); |
||||
|
|
||||
|
@AfterStartUp |
||||
|
public void init() { |
||||
|
DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { |
||||
|
EntitiesVersionControlSettings settings = getSettings(tenantId); |
||||
|
if (settings != null) { |
||||
|
try { |
||||
|
gitRepositoryService.initRepository(tenantId, settings); |
||||
|
} catch (Exception e) { |
||||
|
log.warn("Failed to init repository for tenant {}", tenantId, e); |
||||
|
} |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
@SneakyThrows |
||||
|
public EntitiesVersionControlSettings getSettings(TenantId tenantId) { |
||||
|
return attributesService.find(tenantId, tenantId, DataConstants.SERVER_SCOPE, DefaultEntitiesVersionControlService.SETTINGS_KEY).get() |
||||
|
.flatMap(KvEntry::getJsonValue) |
||||
|
.map(json -> { |
||||
|
try { |
||||
|
return JacksonUtil.fromString(json, EntitiesVersionControlSettings.class); |
||||
|
} catch (IllegalArgumentException e) { |
||||
|
return null; |
||||
|
} |
||||
|
}) |
||||
|
.orElse(null); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { |
||||
|
var lock = getRepoLock(tenantId); |
||||
|
lock.lock(); |
||||
|
try { |
||||
|
gitRepositoryService.initRepository(tenantId, settings); |
||||
|
} catch (Exception e) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(e); |
||||
|
} finally { |
||||
|
lock.unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { |
||||
|
var lock = getRepoLock(tenantId); |
||||
|
lock.lock(); |
||||
|
try { |
||||
|
var pendingCommit = new PendingCommit(tenantId, request); |
||||
|
PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); |
||||
|
if (old != null) { |
||||
|
gitRepositoryService.abort(old); |
||||
|
} |
||||
|
gitRepositoryService.prepareCommit(pendingCommit); |
||||
|
return pendingCommit; |
||||
|
} finally { |
||||
|
lock.unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void deleteAll(PendingCommit commit, EntityType entityType) { |
||||
|
doInsideLock(commit, c -> { |
||||
|
try { |
||||
|
gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); |
||||
|
} catch (IOException e) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(e); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void addToCommit(PendingCommit commit, EntityExportData<ExportableEntity<EntityId>> entityData) { |
||||
|
doInsideLock(commit, c -> { |
||||
|
String entityDataJson; |
||||
|
try { |
||||
|
entityDataJson = jsonWriter.writeValueAsString(entityData); |
||||
|
gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), |
||||
|
entityData.getEntity().getId().toString()), entityDataJson); |
||||
|
} catch (IOException e) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(e); |
||||
|
} |
||||
|
}); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public VersionCreationResult push(PendingCommit commit) { |
||||
|
return executeInsideLock(commit, gitRepositoryService::push); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersion> listVersions(TenantId tenantId, String branch) { |
||||
|
return listVersions(tenantId, branch, (String) null); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityType entityType) { |
||||
|
return listVersions(tenantId, branch, getRelativePath(entityType, null)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityId entityId) { |
||||
|
return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { |
||||
|
try { |
||||
|
return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); |
||||
|
} catch (Exception e) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { |
||||
|
return listEntitiesAtVersion(tenantId, branch, versionId, null); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<String> listBranches(TenantId tenantId) { |
||||
|
return gitRepositoryService.listBranches(tenantId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public EntityExportData<?> getEntity(TenantId tenantId, String versionId, EntityId entityId) { |
||||
|
try { |
||||
|
String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, |
||||
|
getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); |
||||
|
return JacksonUtil.fromString(entityDataJson, EntityExportData.class); |
||||
|
} catch (Exception e) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) { |
||||
|
try { |
||||
|
return gitRepositoryService.listVersions(tenantId, branch, path); |
||||
|
} catch (Exception e) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void doInsideLock(PendingCommit commit, Consumer<PendingCommit> r) { |
||||
|
var lock = getRepoLock(commit.getTenantId()); |
||||
|
lock.lock(); |
||||
|
try { |
||||
|
checkCommit(commit); |
||||
|
r.accept(commit); |
||||
|
} finally { |
||||
|
lock.unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private <T> T executeInsideLock(PendingCommit commit, Function<PendingCommit, T> c) { |
||||
|
var lock = getRepoLock(commit.getTenantId()); |
||||
|
lock.lock(); |
||||
|
try { |
||||
|
checkCommit(commit); |
||||
|
return c.apply(commit); |
||||
|
} finally { |
||||
|
lock.unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void checkCommit(PendingCommit commit) { |
||||
|
PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); |
||||
|
if (existing == null || !existing.getTxId().equals(commit.getTxId())) { |
||||
|
throw new ConcurrentModificationException(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private String getRelativePath(EntityType entityType, String entityId) { |
||||
|
String path = entityType.name().toLowerCase(); |
||||
|
if (entityId != null) { |
||||
|
path += "/" + entityId + ".json"; |
||||
|
} |
||||
|
return path; |
||||
|
} |
||||
|
|
||||
|
private Lock getRepoLock(TenantId tenantId) { |
||||
|
return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,115 @@ |
|||||
|
<!-- |
||||
|
|
||||
|
Copyright © 2016-2022 The Thingsboard Authors |
||||
|
|
||||
|
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
you may not use this file except in compliance with the License. |
||||
|
You may obtain a copy of the License at |
||||
|
|
||||
|
http://www.apache.org/licenses/LICENSE-2.0 |
||||
|
|
||||
|
Unless required by applicable law or agreed to in writing, software |
||||
|
distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
See the License for the specific language governing permissions and |
||||
|
limitations under the License. |
||||
|
|
||||
|
--> |
||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" |
||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> |
||||
|
<modelVersion>4.0.0</modelVersion> |
||||
|
<parent> |
||||
|
<groupId>org.thingsboard</groupId> |
||||
|
<version>3.4.0-SNAPSHOT</version> |
||||
|
<artifactId>common</artifactId> |
||||
|
</parent> |
||||
|
<groupId>org.thingsboard.common</groupId> |
||||
|
<artifactId>version-control</artifactId> |
||||
|
<packaging>jar</packaging> |
||||
|
|
||||
|
<name>Thingsboard Server Version Control API</name> |
||||
|
<url>https://thingsboard.io</url> |
||||
|
|
||||
|
<properties> |
||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> |
||||
|
<main.dir>${basedir}/../..</main.dir> |
||||
|
</properties> |
||||
|
|
||||
|
<dependencies> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework</groupId> |
||||
|
<artifactId>spring-core</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework</groupId> |
||||
|
<artifactId>spring-context-support</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework</groupId> |
||||
|
<artifactId>spring-context</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-web</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>javax.annotation</groupId> |
||||
|
<artifactId>javax.annotation-api</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.google.guava</groupId> |
||||
|
<artifactId>guava</artifactId> |
||||
|
<scope>provided</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>com.fasterxml.jackson.core</groupId> |
||||
|
<artifactId>jackson-databind</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.slf4j</groupId> |
||||
|
<artifactId>slf4j-api</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.slf4j</groupId> |
||||
|
<artifactId>log4j-over-slf4j</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>ch.qos.logback</groupId> |
||||
|
<artifactId>logback-core</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>ch.qos.logback</groupId> |
||||
|
<artifactId>logback-classic</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.eclipse.jgit</groupId> |
||||
|
<artifactId>org.eclipse.jgit</artifactId> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.springframework.boot</groupId> |
||||
|
<artifactId>spring-boot-starter-test</artifactId> |
||||
|
<scope>test</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.junit.vintage</groupId> |
||||
|
<artifactId>junit-vintage-engine</artifactId> |
||||
|
<scope>test</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.awaitility</groupId> |
||||
|
<artifactId>awaitility</artifactId> |
||||
|
<scope>test</scope> |
||||
|
</dependency> |
||||
|
<dependency> |
||||
|
<groupId>org.thingsboard.common</groupId> |
||||
|
<artifactId>data</artifactId> |
||||
|
</dependency> |
||||
|
</dependencies> |
||||
|
|
||||
|
<build> |
||||
|
<plugins> |
||||
|
</plugins> |
||||
|
</build> |
||||
|
|
||||
|
</project> |
||||
@ -0,0 +1,239 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2022 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.sync.vc; |
||||
|
|
||||
|
import lombok.SneakyThrows; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.commons.io.FileUtils; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.eclipse.jgit.api.errors.GitAPIException; |
||||
|
import org.eclipse.jgit.api.errors.JGitInternalException; |
||||
|
import org.eclipse.jgit.api.errors.RefAlreadyExistsException; |
||||
|
import org.springframework.beans.factory.annotation.Value; |
||||
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.EntityIdFactory; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
||||
|
|
||||
|
import javax.annotation.PostConstruct; |
||||
|
import javax.annotation.PreDestroy; |
||||
|
import java.io.File; |
||||
|
import java.io.IOException; |
||||
|
import java.nio.charset.StandardCharsets; |
||||
|
import java.nio.file.Files; |
||||
|
import java.nio.file.Path; |
||||
|
import java.util.List; |
||||
|
import java.util.Map; |
||||
|
import java.util.Optional; |
||||
|
import java.util.concurrent.ConcurrentHashMap; |
||||
|
import java.util.concurrent.Executors; |
||||
|
import java.util.concurrent.ScheduledExecutorService; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
@Slf4j |
||||
|
@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) |
||||
|
@Service |
||||
|
public class DefaultGitRepositoryService implements GitRepositoryService { |
||||
|
|
||||
|
@Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") |
||||
|
private String repositoriesFolder; |
||||
|
|
||||
|
@Value("${vc.git.repos-poll-interval:60}") |
||||
|
private long reposPollInterval; |
||||
|
|
||||
|
private ScheduledExecutorService scheduler; |
||||
|
private final Map<TenantId, GitRepository> repositories = new ConcurrentHashMap<>(); |
||||
|
|
||||
|
@PostConstruct |
||||
|
public void init() { |
||||
|
scheduler = Executors.newSingleThreadScheduledExecutor(); |
||||
|
scheduler.scheduleWithFixedDelay(() -> { |
||||
|
repositories.forEach((tenantId, repository) -> { |
||||
|
try { |
||||
|
repository.fetch(); |
||||
|
log.info("Fetching remote repository for tenant {}", tenantId); |
||||
|
} catch (Exception e) { |
||||
|
log.warn("Failed to fetch repository for tenant {}", tenantId, e); |
||||
|
} |
||||
|
}); |
||||
|
}, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); |
||||
|
} |
||||
|
|
||||
|
@PreDestroy |
||||
|
public void stop() { |
||||
|
if (scheduler != null) { |
||||
|
scheduler.shutdownNow(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void prepareCommit(PendingCommit commit) { |
||||
|
GitRepository repository = checkRepository(commit.getTenantId()); |
||||
|
String branch = commit.getRequest().getBranch(); |
||||
|
try { |
||||
|
repository.fetch(); |
||||
|
if (repository.listBranches().contains(branch)) { |
||||
|
repository.checkout("origin/" + branch, false); |
||||
|
try { |
||||
|
repository.checkout(branch, true); |
||||
|
} catch (RefAlreadyExistsException e) { |
||||
|
repository.checkout(branch, false); |
||||
|
} |
||||
|
repository.merge(branch); |
||||
|
} else { // TODO [viacheslav]: rollback orphan branch on failure
|
||||
|
try { |
||||
|
repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
|
||||
|
} catch (JGitInternalException e) { |
||||
|
if (!e.getMessage().contains("NO_CHANGE")) { |
||||
|
throw e; |
||||
|
} |
||||
|
} |
||||
|
} |
||||
|
} catch (IOException | GitAPIException gitAPIException) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(gitAPIException); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { |
||||
|
GitRepository repository = checkRepository(commit.getTenantId()); |
||||
|
FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { |
||||
|
GitRepository repository = checkRepository(commit.getTenantId()); |
||||
|
FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public VersionCreationResult push(PendingCommit commit) { |
||||
|
GitRepository repository = checkRepository(commit.getTenantId()); |
||||
|
try { |
||||
|
repository.add("."); |
||||
|
|
||||
|
VersionCreationResult result = new VersionCreationResult(); |
||||
|
GitRepository.Status status = repository.status(); |
||||
|
result.setAdded(status.getAdded().size()); |
||||
|
result.setModified(status.getModified().size()); |
||||
|
result.setRemoved(status.getRemoved().size()); |
||||
|
|
||||
|
GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); |
||||
|
repository.push(); |
||||
|
|
||||
|
result.setVersion(toVersion(gitCommit)); |
||||
|
return result; |
||||
|
} catch (GitAPIException gitAPIException) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(gitAPIException); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void abort(PendingCommit commit) { |
||||
|
//TODO: implement;
|
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { |
||||
|
GitRepository repository = checkRepository(tenantId); |
||||
|
return repository.getFileContentAtCommit(relativePath, versionId); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<String> listBranches(TenantId tenantId) { |
||||
|
GitRepository repository = checkRepository(tenantId); |
||||
|
try { |
||||
|
return repository.listBranches(); |
||||
|
} catch (GitAPIException gitAPIException) { |
||||
|
//TODO: analyze and return meaningful exceptions that we can show to the client;
|
||||
|
throw new RuntimeException(gitAPIException); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { |
||||
|
return listVersions(tenantId, branch, null).stream() |
||||
|
.filter(version -> version.getId().equals(versionId)) |
||||
|
.findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); |
||||
|
} |
||||
|
|
||||
|
private GitRepository checkRepository(TenantId tenantId) { |
||||
|
return Optional.ofNullable(repositories.get(tenantId)) |
||||
|
.orElseThrow(() -> new IllegalStateException("Repository is not initialized")); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception { |
||||
|
GitRepository repository = checkRepository(tenantId); |
||||
|
return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() |
||||
|
.map(this::toVersion) |
||||
|
.collect(Collectors.toList()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { |
||||
|
GitRepository repository = checkRepository(tenantId); |
||||
|
checkVersion(tenantId, branch, versionId); |
||||
|
return repository.listFilesAtCommit(versionId, path).stream() |
||||
|
.map(filePath -> { |
||||
|
EntityId entityId = fromRelativePath(filePath); |
||||
|
VersionedEntityInfo info = new VersionedEntityInfo(); |
||||
|
info.setExternalId(entityId); |
||||
|
return info; |
||||
|
}) |
||||
|
.collect(Collectors.toList()); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { |
||||
|
Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); |
||||
|
GitRepository repository; |
||||
|
if (Files.exists(repositoryDirectory)) { |
||||
|
FileUtils.forceDelete(repositoryDirectory.toFile()); |
||||
|
} |
||||
|
|
||||
|
Files.createDirectories(repositoryDirectory); |
||||
|
repository = GitRepository.clone(settings.getRepositoryUri(), settings.getUsername(), settings.getPassword(), repositoryDirectory.toFile()); |
||||
|
repositories.put(tenantId, repository); |
||||
|
} |
||||
|
|
||||
|
private void clearRepository(TenantId tenantId) throws IOException { |
||||
|
GitRepository repository = repositories.get(tenantId); |
||||
|
if (repository != null) { |
||||
|
FileUtils.deleteDirectory(new File(repository.getDirectory())); |
||||
|
repositories.remove(tenantId); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private EntityVersion toVersion(GitRepository.Commit commit) { |
||||
|
return new EntityVersion(commit.getId(), commit.getMessage()); |
||||
|
} |
||||
|
|
||||
|
private EntityId fromRelativePath(String path) { |
||||
|
EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); |
||||
|
String entityId = StringUtils.substringBetween(path, "/", ".json"); |
||||
|
return EntityIdFactory.getByTypeAndUuid(entityType, entityId); |
||||
|
} |
||||
|
} |
||||
@ -0,0 +1,48 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2022 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.sync.vc; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
||||
|
|
||||
|
import java.io.IOException; |
||||
|
import java.util.List; |
||||
|
|
||||
|
public interface GitRepositoryService { |
||||
|
|
||||
|
void prepareCommit(PendingCommit pendingCommit); |
||||
|
|
||||
|
List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception; |
||||
|
|
||||
|
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; |
||||
|
|
||||
|
void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; |
||||
|
|
||||
|
void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; |
||||
|
|
||||
|
void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; |
||||
|
|
||||
|
VersionCreationResult push(PendingCommit commit); |
||||
|
|
||||
|
void abort(PendingCommit commit); |
||||
|
|
||||
|
List<String> listBranches(TenantId tenantId); |
||||
|
|
||||
|
String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; |
||||
|
} |
||||
@ -0,0 +1,60 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2022 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.sync.vc; |
||||
|
|
||||
|
import lombok.SneakyThrows; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.ExportableEntity; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.sync.ie.EntityExportData; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; |
||||
|
import org.thingsboard.server.common.data.sync.vc.EntityVersion; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; |
||||
|
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; |
||||
|
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
public interface GitVersionControlService { |
||||
|
|
||||
|
@SneakyThrows |
||||
|
EntitiesVersionControlSettings getSettings(TenantId tenantId); |
||||
|
|
||||
|
void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); |
||||
|
|
||||
|
PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); |
||||
|
|
||||
|
void addToCommit(PendingCommit commit, EntityExportData<ExportableEntity<EntityId>> entityData); |
||||
|
|
||||
|
void deleteAll(PendingCommit pendingCommit, EntityType entityType); |
||||
|
|
||||
|
VersionCreationResult push(PendingCommit commit); |
||||
|
|
||||
|
List<EntityVersion> listVersions(TenantId tenantId, String branch); |
||||
|
|
||||
|
List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityType entityType); |
||||
|
|
||||
|
List<EntityVersion> listVersions(TenantId tenantId, String branch, EntityId entityId); |
||||
|
|
||||
|
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); |
||||
|
|
||||
|
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); |
||||
|
|
||||
|
List<String> listBranches(TenantId tenantId); |
||||
|
|
||||
|
EntityExportData<?> getEntity(TenantId tenantId, String versionId, EntityId entityId); |
||||
|
} |
||||
@ -0,0 +1,36 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2022 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.sync.vc; |
||||
|
|
||||
|
import lombok.Data; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; |
||||
|
|
||||
|
import java.util.UUID; |
||||
|
|
||||
|
@Data |
||||
|
public class PendingCommit { |
||||
|
|
||||
|
private final UUID txId; |
||||
|
private final TenantId tenantId; |
||||
|
private final VersionCreateRequest request; |
||||
|
|
||||
|
public PendingCommit(TenantId tenantId, VersionCreateRequest request) { |
||||
|
this.txId = UUID.randomUUID(); |
||||
|
this.tenantId = tenantId; |
||||
|
this.request = request; |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue