tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Merge pull request #6585 from ViacheslavKlimov/entities-vc-refactoring 

Git api usage improvements
pull/6588/head
Igor Kulikov 4 years ago
committed by GitHub
parent
421624f3d4 2a89fc67ae
commit
490a0c2ae3
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 93 additions and 60 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 29
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  2. 20
      application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java
  3. 1
      application/src/main/resources/thingsboard.yml
  4. 3
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java
  5. 51
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
  6. 45
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
  7. 2
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
  8. 2
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java

29
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -24,37 +24,39 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.AdminSettings;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.*;
import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.entity.EntityService;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.sync.ThrowingRunnable;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.settings.AdminSettingsService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import java.util.ArrayList;
import java.util.HashMap;
@ -76,7 +78,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private final EntitiesExportImportService exportImportService;
private final ExportableEntitiesService exportableEntitiesService;
private final AdminSettingsService adminSettingsService;
private final EntityService entityService;
private final TransactionTemplate transactionTemplate;
@Override

20
application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java
View File

@ -135,7 +135,13 @@ public class LocalGitVersionControlService implements GitVersionControlService {
if (old != null) {
gitRepositoryService.abort(old);
}
gitRepositoryService.prepareCommit(pendingCommit);
try {
gitRepositoryService.prepareCommit(pendingCommit);
} catch (Exception e) {
pendingCommitMap.remove(tenantId);
gitRepositoryService.cleanUp(pendingCommit);
throw e;
}
return pendingCommit;
} finally {
lock.unlock();
@ -171,7 +177,9 @@ public class LocalGitVersionControlService implements GitVersionControlService {
@Override
public VersionCreationResult push(PendingCommit commit) {
return executeInsideLock(commit, gitRepositoryService::push);
VersionCreationResult result = executeInsideLock(commit, gitRepositoryService::push);
pendingCommitMap.remove(commit.getTenantId());
return result;
}
@Override
@ -256,6 +264,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
try {
checkCommit(commit);
r.accept(commit);
} catch (Exception e) {
pendingCommitMap.remove(commit.getTenantId());
gitRepositoryService.cleanUp(commit);
throw e;
} finally {
lock.unlock();
}
@ -267,6 +279,10 @@ public class LocalGitVersionControlService implements GitVersionControlService {
try {
checkCommit(commit);
return c.apply(commit);
} catch (Exception e) {
pendingCommitMap.remove(commit.getTenantId());
gitRepositoryService.cleanUp(commit);
throw e;
} finally {
lock.unlock();
}

1
application/src/main/resources/thingsboard.yml
View File

@ -1119,6 +1119,7 @@ vc:
git:
service: "${JS_VC_GIT_SERVICE:local}" # local/remote
repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}"
repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}"
management:
endpoints:

3
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java
View File

@ -16,6 +16,7 @@
package org.thingsboard.server.common.data.sync.vc;
import lombok.Data;
@Data
public class EntitiesVersionControlSettings {
private String repositoryUri;
@ -26,4 +27,4 @@ public class EntitiesVersionControlSettings {
private String privateKey;
private String privateKeyPassword;
private String defaultBranch;
}
}

51
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
View File

@ -20,8 +20,6 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.api.errors.JGitInternalException;
import org.eclipse.jgit.api.errors.RefAlreadyExistsException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@ -55,7 +53,7 @@ import java.util.stream.Collectors;
@Service
public class DefaultGitRepositoryService implements GitRepositoryService {
@Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}")
@Value("${vc.git.repositories-folder}")
private String repositoriesFolder;
@Value("${vc.git.repos-poll-interval:60}")
@ -92,22 +90,12 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
String branch = commit.getRequest().getBranch();
try {
repository.fetch();
if (repository.listBranches().contains(branch)) {
repository.checkout("origin/" + branch, false);
try {
repository.checkout(branch, true);
} catch (RefAlreadyExistsException e) {
repository.checkout(branch, false);
}
repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
repository.resetAndClean();
if (repository.listRemoteBranches().contains(branch)) {
repository.merge(branch);
} else { // TODO [viacheslav]: rollback orphan branch on failure
try {
repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch
} catch (JGitInternalException e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
}
} catch (IOException | GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
@ -140,32 +128,49 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
result.setRemoved(status.getRemoved().size());
GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName());
repository.push();
repository.push(commit.getWorkingBranch(), commit.getRequest().getBranch());
result.setVersion(toVersion(gitCommit));
return result;
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);
} finally {
cleanUp(commit);
}
}
@SneakyThrows
@Override
public void cleanUp(PendingCommit commit) {
GitRepository repository = checkRepository(commit.getTenantId());
try {
repository.createAndCheckoutOrphanBranch(EntityId.NULL_UUID.toString());
} catch (Exception e) {
if (!e.getMessage().contains("NO_CHANGE")) {
throw e;
}
}
repository.resetAndClean();
repository.deleteLocalBranchIfExists(commit.getWorkingBranch());
}
@Override
public void abort(PendingCommit commit) {
//TODO: implement;
cleanUp(commit);
}
@Override
public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException {
GitRepository repository = checkRepository(tenantId);
return repository.getFileContentAtCommit(relativePath, versionId);
GitRepository repository = checkRepository(tenantId);
return repository.getFileContentAtCommit(relativePath, versionId);
}
@Override
public List<String> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId);
try {
return repository.listBranches();
return repository.listRemoteBranches();
} catch (GitAPIException gitAPIException) {
//TODO: analyze and return meaningful exceptions that we can show to the client;
throw new RuntimeException(gitAPIException);

45
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
View File

@ -18,8 +18,16 @@ package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Streams;
import lombok.Data;
import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.sshd.common.util.security.SecurityUtils;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.CloneCommand;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.GitCommand;
import org.eclipse.jgit.api.ListBranchCommand;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.LsRemoteCommand;
import org.eclipse.jgit.api.ResetCommand;
import org.eclipse.jgit.api.TransportCommand;
import org.eclipse.jgit.api.errors.GitAPIException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
@ -28,6 +36,7 @@ import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
import org.eclipse.jgit.transport.sshd.JGitKeyCache;
@ -36,7 +45,6 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactory;
import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder;
import org.eclipse.jgit.treewalk.TreeWalk;
import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings;
import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod;
@ -116,15 +124,18 @@ public class GitRepository {
.setRemoveDeletedRefs(true));
}
public void checkout(String branch, boolean createBranch) throws GitAPIException {
execute(git.checkout()
.setCreateBranch(createBranch)
.setName(branch));
public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
execute(git.branchDelete()
.setBranchNames(branch)
.setForce(true));
}
public void reset() throws GitAPIException {
public void resetAndClean() throws GitAPIException {
execute(git.reset()
.setMode(ResetCommand.ResetType.HARD));
execute(git.clean()
.setForce(true)
.setCleanDirectories(true));
}
public void merge(String branch) throws IOException, GitAPIException {
@ -137,9 +148,9 @@ public class GitRepository {
}
public List<String> listBranches() throws GitAPIException {
public List<String> listRemoteBranches() throws GitAPIException {
return execute(git.branchList()
.setListMode(ListBranchCommand.ListMode.ALL)).stream()
.setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
.filter(ref -> !ref.getName().equals(Constants.HEAD))
.map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
.map(name -> StringUtils.removeStart(name, "origin/"))
@ -209,16 +220,9 @@ public class GitRepository {
.setOrphan(true)
.setForced(true)
.setName(name));
// Set<String> uncommittedChanges = git.status().call().getUncommittedChanges();
// if (!uncommittedChanges.isEmpty()) {
// RmCommand rm = git.rm();
// uncommittedChanges.forEach(rm::addFilepattern);
// execute(rm);
// }
// execute(git.clean());
}
public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav]
public void add(String filesPattern) throws GitAPIException {
execute(git.add().setUpdate(true).addFilepattern(filesPattern));
execute(git.add().addFilepattern(filesPattern));
}
@ -230,13 +234,14 @@ public class GitRepository {
public Commit commit(String message) throws GitAPIException {
RevCommit revCommit = execute(git.commit()
.setMessage(message)); // TODO [viacheslav]: set configurable author for commit
.setMessage(message));
return toCommit(revCommit);
}
public void push() throws GitAPIException {
execute(git.push());
public void push(String localBranch, String remoteBranch) throws GitAPIException {
execute(git.push()
.setRefSpecs(new RefSpec(localBranch + ":" + remoteBranch)));
}

2
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
View File

@ -44,6 +44,8 @@ public interface GitRepositoryService {
VersionCreationResult push(PendingCommit commit);
void cleanUp(PendingCommit commit);
void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId);

2
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java
View File

@ -27,10 +27,12 @@ public class PendingCommit {
private final UUID txId;
private final TenantId tenantId;
private final VersionCreateRequest request;
private final String workingBranch;
public PendingCommit(TenantId tenantId, VersionCreateRequest request) {
this.txId = UUID.randomUUID();
this.tenantId = tenantId;
this.request = request;
this.workingBranch = txId.toString();
}
}

Write Preview
Loading…
Cancel
Save