Introduce Git sync service

application/src/main/java/org/thingsboard/server/service/sync/DefaultGitSyncService.java

@@ -0,0 +1,164 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync;
+
+import jakarta.annotation.PreDestroy;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.thingsboard.common.util.ThingsBoardThreadFactory;
+import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
+import org.thingsboard.server.service.sync.vc.GitRepository;
+import org.thingsboard.server.service.sync.vc.GitRepository.FileType;
+import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile;
+
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class DefaultGitSyncService implements GitSyncService {
+
+    @Value("${vc.git.repositories-folder:${java.io.tmpdir}/repositories}")
+    private String repositoriesFolder;
+
+    private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("git-sync"));
+    private final Map<String, GitRepository> repositories = new ConcurrentHashMap<>();
+    private final Map<String, Runnable> updateListeners = new ConcurrentHashMap<>();
+
+    @Override
+    public void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate) {
+        RepositorySettings settings = new RepositorySettings();
+        settings.setRepositoryUri(repoUri);
+        settings.setDefaultBranch(branch);
+        if (onUpdate != null) {
+            updateListeners.put(key, onUpdate);
+        }
+
+        executor.execute(() -> {
+            initRepository(key, settings);
+        });
+
+        executor.scheduleWithFixedDelay(() -> {
+            GitRepository repository = repositories.get(key);
+            if (repository == null || Files.notExists(Path.of(repository.getDirectory()))) {
+                initRepository(key, settings);
+                return;
+            }
+
+            try {
+                log.debug("Fetching {} repository", key);
+                repository.fetch();
+                onUpdate(key);
+            } catch (Throwable e) {
+                log.error("Failed to fetch {} repository", key, e);
+            }
+        }, fetchFrequencyMs, fetchFrequencyMs, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public List<RepoFile> listFiles(String key, String path, int depth, FileType type) {
+        GitRepository repository = getRepository(key);
+        return repository.listFilesAtCommit(getBranchRef(repository), path, depth).stream()
+                .filter(file -> type == null || file.type() == type)
+                .toList();
+    }
+
+
+    @Override
+    public String getFileContent(String key, String path) {
+        GitRepository repository = getRepository(key);
+        try {
+            return repository.getFileContentAtCommit(path, getBranchRef(repository));
+        } catch (Exception e) {
+            log.warn("Failed to get file content for path {} for {} repository: {}", path, key, e.getMessage());
+            return "{}";
+        }
+    }
+
+    @Override
+    public String getGithubRawContentUrl(String key, String path) {
+        if (path == null) {
+            return "";
+        }
+        RepositorySettings settings = getRepository(key).getSettings();
+        return StringUtils.removeEnd(settings.getRepositoryUri(), ".git") + "/blob/" + settings.getDefaultBranch() + "/" + path + "?raw=true";
+    }
+
+    private GitRepository getRepository(String key) {
+        GitRepository repository = repositories.get(key);
+        if (repository != null) {
+            if (Files.notExists(Path.of(repository.getDirectory()))) {
+                // reinitializing the repository because folder was deleted
+                initRepository(key, repository.getSettings());
+            }
+        }
+
+        repository = repositories.get(key);
+        if (repository == null) {
+            throw new IllegalStateException("{} repository is not initialized");
+        }
+        return repository;
+    }
+
+    private void initRepository(String key, RepositorySettings settings) {
+        try {
+            repositories.remove(key);
+            Path directory = getRepoDirectory(settings);
+
+            GitRepository repository = GitRepository.openOrClone(directory, settings, true);
+            repositories.put(key, repository);
+            log.info("Initialized {} repository", key);
+
+            onUpdate(key);
+        } catch (Throwable e) {
+            log.error("Failed to init {} repository for settings {}", key, settings, e);
+        }
+    }
+
+    private void onUpdate(String key) {
+        Runnable listener = updateListeners.get(key);
+        if (listener != null) {
+            listener.run();
+        }
+    }
+
+    private Path getRepoDirectory(RepositorySettings settings) {
+        // using uri to define folder name in case repo url is changed
+        String name = URI.create(settings.getRepositoryUri()).getPath().replaceAll("[^a-zA-Z]", "");
+        return Path.of(repositoriesFolder, name);
+    }
+
+    private String getBranchRef(GitRepository repository) {
+        return "refs/remotes/origin/" + repository.getSettings().getDefaultBranch();
+    }
+
+    @PreDestroy
+    private void preDestroy() {
+        executor.shutdownNow();
+    }
+
+}

application/src/main/java/org/thingsboard/server/service/sync/GitSyncService.java

@@ -0,0 +1,33 @@
+/**
+ * Copyright © 2016-2024 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync;
+
+import org.thingsboard.server.service.sync.vc.GitRepository.FileType;
+import org.thingsboard.server.service.sync.vc.GitRepository.RepoFile;
+
+import java.util.List;
+
+public interface GitSyncService {
+
+    void registerSync(String key, String repoUri, String branch, long fetchFrequencyMs, Runnable onUpdate);
+
+    List<RepoFile> listFiles(String key, String path, int depth, FileType type);
+
+    String getFileContent(String key, String path);
+
+    String getGithubRawContentUrl(String key, String path);
+
+}

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java

@@ -283,22 +283,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
     private GitRepository openOrCloneRepository(TenantId tenantId, RepositorySettings settings, boolean fetch) throws Exception {
         log.debug("[{}] Init tenant repository started.", tenantId);
         Path repositoryDirectory = Path.of(repositoriesFolder, settings.isLocalOnly() ? "local_" + settings.getRepositoryUri() : tenantId.getId().toString());
-
-        GitRepository repository;
-        if (Files.exists(repositoryDirectory)) {
-            repository = GitRepository.open(repositoryDirectory.toFile(), settings);
-            if (fetch) {
-                repository.fetch();
-            }
-        } else {
-            Files.createDirectories(repositoryDirectory);
-            if (settings.isLocalOnly()) {
-                repository = GitRepository.create(settings, repositoryDirectory.toFile());
-            } else {
-                repository = GitRepository.clone(settings, repositoryDirectory.toFile());
-            }
-        }
-
+        GitRepository repository = GitRepository.openOrClone(repositoryDirectory, settings, fetch);
         repositories.put(tenantId, repository);
         log.debug("[{}] Init tenant repository completed.", tenantId);
         return repository;

common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@@ -21,6 +21,7 @@ import com.google.common.collect.Streams;
 import lombok.Data;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.StringUtils;
@@ -75,6 +76,7 @@ import java.io.IOException;
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.security.KeyPair;
 import java.security.PublicKey;
 import java.util.ArrayList;
@@ -136,6 +138,24 @@ public class GitRepository {
         return new GitRepository(git, settings, authHandler, directory.getAbsolutePath());
     }
 
+    public static GitRepository openOrClone(Path directory, RepositorySettings settings, boolean fetch) throws IOException, GitAPIException {
+        GitRepository repository;
+        if (Files.exists(directory)) {
+            repository = GitRepository.open(directory.toFile(), settings);
+            if (fetch) {
+                repository.fetch();
+            }
+        } else {
+            Files.createDirectories(directory);
+            if (settings.isLocalOnly()) {
+                repository = GitRepository.create(settings, directory.toFile());
+            } else {
+                repository = GitRepository.clone(settings, directory.toFile());
+            }
+        }
+        return repository;
+    }
+
     public static void test(RepositorySettings settings, File directory) throws Exception {
         if (settings.isLocalOnly()) {
             return;
@@ -233,22 +253,29 @@ public class GitRepository {
         return iterableToPageData(commits, this::toCommit, pageLink, revCommitComparatorFunction);
     }
 
-    public List<String> listFilesAtCommit(String commitId) throws IOException {
-        return listFilesAtCommit(commitId, null);
+    public List<String> listFilesAtCommit(String commitId, String path) {
+        return listFilesAtCommit(commitId, path, -1).stream().map(RepoFile::path).toList();
     }
 
-    public List<String> listFilesAtCommit(String commitId, String path) throws IOException {
+    @SneakyThrows
+    public List<RepoFile> listFilesAtCommit(String commitId, String path, int depth) {
         log.debug("Executing listFilesAtCommit [{}][{}][{}]", settings.getRepositoryUri(), commitId, path);
-        List<String> files = new ArrayList<>();
+        List<RepoFile> files = new ArrayList<>();
         RevCommit revCommit = resolveCommit(commitId);
         try (TreeWalk treeWalk = new TreeWalk(git.getRepository())) {
             treeWalk.reset(revCommit.getTree().getId());
             if (StringUtils.isNotEmpty(path)) {
                 treeWalk.setFilter(PathFilter.create(path));
             }
-            treeWalk.setRecursive(true);
+            boolean fixedDepth = depth != -1;
+            treeWalk.setRecursive(!fixedDepth);
             while (treeWalk.next()) {
-                files.add(treeWalk.getPathString());
+                if (!fixedDepth || treeWalk.getDepth() == depth) {
+                    files.add(new RepoFile(treeWalk.getPathString(), treeWalk.getNameString(), treeWalk.isSubtree() ? FileType.DIRECTORY : FileType.FILE));
+                }
+                if (fixedDepth && treeWalk.getDepth() < depth) {
+                    treeWalk.enterSubtree();
+                }
             }
         }
         return files;
@@ -584,4 +611,10 @@ public class GitRepository {
         private String diffStringValue;
     }
 
+    public record RepoFile(String path, String name, FileType type) {}
+
+    public enum FileType {
+        FILE, DIRECTORY
+    }
+
 }