Viacheslav Klimov
4 years ago
12 changed files with 665 additions and 632 deletions
@ -0,0 +1,395 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import com.fasterxml.jackson.databind.ObjectMapper; |
|||
import com.fasterxml.jackson.databind.ObjectWriter; |
|||
import com.fasterxml.jackson.databind.SerializationFeature; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.SneakyThrows; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.io.FileUtils; |
|||
import org.apache.commons.lang3.StringUtils; |
|||
import org.eclipse.jgit.api.errors.GitAPIException; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.AdminSettings; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.ExportableEntity; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.EntityIdFactory; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.dao.settings.AdminSettingsService; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
import org.thingsboard.server.service.sync.EntitiesExportImportService; |
|||
import org.thingsboard.server.service.sync.exporting.data.EntityExportData; |
|||
import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; |
|||
import org.thingsboard.server.service.sync.importing.data.EntityImportResult; |
|||
import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; |
|||
import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; |
|||
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersion; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; |
|||
import org.thingsboard.server.utils.GitRepository; |
|||
|
|||
import java.io.File; |
|||
import java.io.IOException; |
|||
import java.nio.charset.StandardCharsets; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.util.ArrayList; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
import java.util.concurrent.Executors; |
|||
import java.util.concurrent.ScheduledExecutorService; |
|||
import java.util.concurrent.ScheduledFuture; |
|||
import java.util.concurrent.TimeUnit; |
|||
import java.util.concurrent.locks.ReadWriteLock; |
|||
import java.util.concurrent.locks.ReentrantReadWriteLock; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Service |
|||
@TbCoreComponent |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { |
|||
|
|||
private final EntitiesExportImportService exportImportService; |
|||
|
|||
private GitRepository repository; |
|||
private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); |
|||
|
|||
private ScheduledExecutorService fetchExecutor; |
|||
private ScheduledFuture<?> fetchTask; |
|||
|
|||
private final AdminSettingsService adminSettingsService; |
|||
private static final String SETTINGS_KEY = "vc"; |
|||
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); |
|||
|
|||
|
|||
@AfterStartUp |
|||
public void init() { |
|||
EntitiesVersionControlSettings settings = getSettings(); |
|||
if (settings != null) { |
|||
try { |
|||
initRepository(settings); |
|||
} catch (Exception e) { |
|||
log.debug("Failed to init repository", e); |
|||
} |
|||
} |
|||
|
|||
int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); |
|||
fetchExecutor = Executors.newSingleThreadScheduledExecutor(); |
|||
fetchTask = scheduleFetch(fetchPeriod); |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { |
|||
return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); |
|||
} |
|||
|
|||
@Override |
|||
public EntityVersion saveEntitiesVersion(SecurityUser user, List<EntityId> entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { |
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
checkRepository(); |
|||
checkBranch(user.getTenantId(), branch); |
|||
|
|||
List<EntityExportData<?>> entityDataList = new ArrayList<>(); |
|||
EntityExportSettings exportSettings = EntityExportSettings.builder() |
|||
.exportRelations(settings.isSaveRelations()) |
|||
.build(); |
|||
for (EntityId entityId : entitiesIds) { |
|||
EntityExportData<ExportableEntity<EntityId>> entityData = exportImportService.exportEntity(user, entityId, exportSettings); |
|||
entityDataList.add(entityData); |
|||
} |
|||
|
|||
fetch(); |
|||
if (repository.listBranches().contains(branch)) { |
|||
repository.checkout(branch); |
|||
repository.merge(branch); |
|||
} else { |
|||
repository.createAndCheckoutOrphanBranch(branch); |
|||
} |
|||
|
|||
for (EntityExportData<?> entityData : entityDataList) { |
|||
String entityDataJson = jsonWriter.writeValueAsString(entityData); |
|||
FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), |
|||
entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); |
|||
} |
|||
|
|||
GitRepository.Commit commit = repository.commit(versionName, "."); |
|||
repository.push(); |
|||
return toVersion(commit); |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public List<EntityVersion> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { |
|||
return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); |
|||
} |
|||
|
|||
@Override |
|||
public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { |
|||
return listVersions(tenantId, branch, getRelativePath(entityType, null)); |
|||
} |
|||
|
|||
@Override |
|||
public List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception { |
|||
return listVersions(tenantId, branch, null); |
|||
} |
|||
|
|||
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
checkRepository(); |
|||
checkBranch(tenantId, branch); |
|||
|
|||
return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() |
|||
.map(this::toVersion) |
|||
.collect(Collectors.toList()); |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { |
|||
return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); |
|||
} |
|||
|
|||
@Override |
|||
public List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { |
|||
return listEntitiesAtVersion(tenantId, branch, versionId, null); |
|||
} |
|||
|
|||
private List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
checkRepository(); |
|||
checkBranch(tenantId, branch); |
|||
checkVersion(tenantId, branch, versionId, path); |
|||
|
|||
return repository.listFilesAtCommit(versionId, path).stream() |
|||
.map(filePath -> { |
|||
EntityId entityId = fromRelativePath(filePath); |
|||
EntityExportData<?> entityData = getEntityDataAtVersion(entityId, versionId); |
|||
|
|||
VersionedEntityInfo info = new VersionedEntityInfo(); |
|||
info.setExternalId(entityId); |
|||
info.setEntityName(entityData.getEntity().getName()); |
|||
return info; |
|||
}) |
|||
.collect(Collectors.toList()); |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { |
|||
return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); |
|||
} |
|||
|
|||
@Override |
|||
public List<EntityVersionLoadResult> loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { |
|||
return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); |
|||
} |
|||
|
|||
@Override |
|||
public List<EntityVersionLoadResult> loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { |
|||
return loadAtVersion(user, branch, versionId, null, settings); |
|||
} |
|||
|
|||
private List<EntityVersionLoadResult> loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { |
|||
List<EntityExportData<?>> entityDataList = new ArrayList<>(); |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { |
|||
EntityExportData<?> entityData = getEntityDataAtVersion(info.getExternalId(), versionId); |
|||
entityDataList.add(entityData); |
|||
} |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
|
|||
EntityImportSettings importSettings = EntityImportSettings.builder() |
|||
.updateRelations(settings.isLoadRelations()) |
|||
.findExistingByName(settings.isFindExistingEntityByName()) |
|||
.build(); |
|||
List<EntityImportResult<?>> importResults = exportImportService.importEntities(user, entityDataList, importSettings); |
|||
|
|||
return importResults.stream() |
|||
.map(importResult -> EntityVersionLoadResult.builder() |
|||
.previousEntityVersion(importResult.getOldEntity()) |
|||
.newEntityVersion(importResult.getSavedEntity()) |
|||
.entityType(importResult.getEntityType()) |
|||
.build()) |
|||
.collect(Collectors.toList()); |
|||
} |
|||
|
|||
@SneakyThrows |
|||
private EntityExportData<?> getEntityDataAtVersion(EntityId externalId, String versionId) { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); |
|||
return JacksonUtil.fromString(entityDataJson, EntityExportData.class); |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
private void fetch() throws GitAPIException { |
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
repository.fetch(); |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
private ScheduledFuture<?> scheduleFetch(int fetchPeriod) { |
|||
return fetchExecutor.scheduleWithFixedDelay(() -> { |
|||
if (repository == null) return; |
|||
try { |
|||
fetch(); |
|||
} catch (Exception e) { |
|||
log.error("Failed to fetch remote repository", e); |
|||
} |
|||
}, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); |
|||
} |
|||
|
|||
|
|||
private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { |
|||
if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { |
|||
throw new IllegalArgumentException("Version not found"); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public List<String> listAllowedBranches(TenantId tenantId) { |
|||
return Optional.ofNullable(getSettings()) |
|||
.flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) |
|||
.flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) |
|||
.orElse(Collections.emptyList()); |
|||
} |
|||
|
|||
private void checkBranch(TenantId tenantId, String branch) { |
|||
if (!listAllowedBranches(tenantId).contains(branch)) { |
|||
throw new IllegalArgumentException("Tenant does not have access to the branch"); |
|||
} |
|||
} |
|||
|
|||
|
|||
private void checkRepository() { |
|||
if (repository == null) { |
|||
throw new IllegalStateException("Repository is not initialized"); |
|||
} |
|||
} |
|||
|
|||
private void initRepository(EntitiesVersionControlSettings settings) throws Exception { |
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { |
|||
this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); |
|||
} else { |
|||
Files.createDirectories(Path.of(settings.getRepositoryDirectory())); |
|||
this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), |
|||
settings.getUsername(), settings.getPassword()); |
|||
} |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
private void clearRepository() throws IOException { |
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
if (repository != null) { |
|||
FileUtils.deleteDirectory(new File(repository.getDirectory())); |
|||
repository = null; |
|||
} |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
@SneakyThrows |
|||
@Override |
|||
public void saveSettings(EntitiesVersionControlSettings settings) { |
|||
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) |
|||
.orElseGet(() -> { |
|||
AdminSettings newAdminSettings = new AdminSettings(); |
|||
newAdminSettings.setKey(SETTINGS_KEY); |
|||
return newAdminSettings; |
|||
}); |
|||
adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); |
|||
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); |
|||
|
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
clearRepository(); |
|||
initRepository(settings); |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
|
|||
if (settings.getFetchPeriod() != 0) { |
|||
fetchTask.cancel(true); |
|||
fetchTask = scheduleFetch(settings.getFetchPeriod()); |
|||
} |
|||
} |
|||
|
|||
@Override |
|||
public EntitiesVersionControlSettings getSettings() { |
|||
return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) |
|||
.map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) |
|||
.orElse(null); |
|||
} |
|||
|
|||
|
|||
private EntityVersion toVersion(GitRepository.Commit commit) { |
|||
return new EntityVersion(commit.getId(), commit.getMessage()); |
|||
} |
|||
|
|||
private String getRelativePath(EntityType entityType, String entityId) { |
|||
String path = entityType.name().toLowerCase(); |
|||
if (entityId != null) { |
|||
path += "/" + entityId + ".json"; |
|||
} |
|||
return path; |
|||
} |
|||
|
|||
private EntityId fromRelativePath(String path) { |
|||
EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); |
|||
String entityId = StringUtils.substringBetween(path, "/", ".json"); |
|||
return EntityIdFactory.getByTypeAndUuid(entityType, entityId); |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,64 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vc; |
|||
|
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.service.security.model.SecurityUser; |
|||
import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; |
|||
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersion; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; |
|||
import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; |
|||
|
|||
import java.util.List; |
|||
|
|||
public interface EntitiesVersionControlService { |
|||
|
|||
EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; |
|||
|
|||
EntityVersion saveEntitiesVersion(SecurityUser user, List<EntityId> entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; |
|||
|
|||
|
|||
List<EntityVersion> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; |
|||
|
|||
List<EntityVersion> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; |
|||
|
|||
List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception; |
|||
|
|||
|
|||
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also
|
|||
|
|||
List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; |
|||
|
|||
|
|||
EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; |
|||
|
|||
List<EntityVersionLoadResult> loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; |
|||
|
|||
List<EntityVersionLoadResult> loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; |
|||
|
|||
|
|||
List<String> listAllowedBranches(TenantId tenantId); |
|||
|
|||
|
|||
void saveSettings(EntitiesVersionControlSettings settings); |
|||
|
|||
EntitiesVersionControlSettings getSettings(); |
|||
|
|||
} |
|||
@ -1,358 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vcs; |
|||
|
|||
import com.fasterxml.jackson.core.type.TypeReference; |
|||
import com.fasterxml.jackson.databind.ObjectMapper; |
|||
import com.fasterxml.jackson.databind.ObjectWriter; |
|||
import com.fasterxml.jackson.databind.SerializationFeature; |
|||
import lombok.RequiredArgsConstructor; |
|||
import lombok.extern.slf4j.Slf4j; |
|||
import org.apache.commons.io.FileUtils; |
|||
import org.eclipse.jgit.api.errors.GitAPIException; |
|||
import org.springframework.scheduling.annotation.Scheduled; |
|||
import org.springframework.stereotype.Service; |
|||
import org.thingsboard.common.util.JacksonUtil; |
|||
import org.thingsboard.server.common.data.AdminSettings; |
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.ExportableEntity; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.dao.settings.AdminSettingsService; |
|||
import org.thingsboard.server.dao.tenant.TenantService; |
|||
import org.thingsboard.server.queue.util.AfterStartUp; |
|||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|||
import org.thingsboard.server.service.sync.EntitiesExportImportService; |
|||
import org.thingsboard.server.service.sync.exporting.EntityExportSettings; |
|||
import org.thingsboard.server.service.sync.exporting.data.EntityExportData; |
|||
import org.thingsboard.server.service.sync.importing.EntityImportResult; |
|||
import org.thingsboard.server.service.sync.importing.EntityImportSettings; |
|||
import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; |
|||
import org.thingsboard.server.service.sync.vcs.data.EntityVersion; |
|||
import org.thingsboard.server.service.sync.vcs.data.GitSettings; |
|||
import org.thingsboard.server.utils.git.Repository; |
|||
import org.thingsboard.server.utils.git.data.Commit; |
|||
|
|||
import java.io.File; |
|||
import java.io.IOException; |
|||
import java.io.UncheckedIOException; |
|||
import java.nio.charset.StandardCharsets; |
|||
import java.nio.file.Files; |
|||
import java.nio.file.Path; |
|||
import java.util.Collections; |
|||
import java.util.List; |
|||
import java.util.Optional; |
|||
import java.util.Set; |
|||
import java.util.concurrent.locks.Lock; |
|||
import java.util.concurrent.locks.ReadWriteLock; |
|||
import java.util.concurrent.locks.ReentrantLock; |
|||
import java.util.concurrent.locks.ReentrantReadWriteLock; |
|||
import java.util.stream.Collectors; |
|||
|
|||
@Service |
|||
@TbCoreComponent |
|||
@RequiredArgsConstructor |
|||
@Slf4j |
|||
public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { |
|||
// TODO [viacheslav]: start up only on one of the cores
|
|||
|
|||
private final TenantService tenantService; |
|||
private final EntitiesExportImportService exportImportService; |
|||
private final AdminSettingsService adminSettingsService; |
|||
|
|||
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); |
|||
private static final String SETTINGS_KEY = "vc"; |
|||
|
|||
private Repository repository; |
|||
private final Lock fetchLock = new ReentrantLock(); |
|||
private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); |
|||
|
|||
@AfterStartUp |
|||
public void init() throws Exception { |
|||
try { |
|||
EntitiesVersionControlSettings settings = getSettings(); |
|||
if (settings != null && settings.getGitSettings() != null) { |
|||
this.repository = initRepository(settings.getGitSettings()); |
|||
} |
|||
} catch (Exception e) { |
|||
log.error("Failed to initialize entities version control service", e); |
|||
} |
|||
} |
|||
|
|||
|
|||
@Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) |
|||
private void fetch() throws Exception { |
|||
if (repository == null) return; |
|||
tryFetch(); |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { |
|||
return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); |
|||
} |
|||
|
|||
@Override |
|||
public EntityVersion saveEntitiesVersion(TenantId tenantId, List<EntityId> entitiesIds, String branch, String versionName) throws Exception { |
|||
checkRepository(); |
|||
checkBranch(tenantId, branch); |
|||
|
|||
EntityExportSettings exportSettings = EntityExportSettings.builder() |
|||
.exportInboundRelations(false) |
|||
.exportOutboundRelations(false) |
|||
.build(); |
|||
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = entitiesIds.stream() |
|||
.map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) |
|||
.collect(Collectors.toList()); |
|||
|
|||
tryFetch(); |
|||
|
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
if (repository.listBranches().contains(branch)) { |
|||
repository.checkout(branch); |
|||
repository.merge(branch); |
|||
} else { |
|||
repository.createAndCheckoutOrphanBranch(branch); |
|||
} |
|||
|
|||
for (EntityExportData<ExportableEntity<EntityId>> entityData : entityDataList) { |
|||
String entityDataJson = jsonWriter.writeValueAsString(entityData); |
|||
FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), |
|||
entityDataJson, StandardCharsets.UTF_8); |
|||
} |
|||
|
|||
Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); |
|||
repository.push(); |
|||
return toVersion(commit); |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
|
|||
@Override |
|||
public List<EntityVersion> listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { |
|||
return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); |
|||
} |
|||
|
|||
@Override |
|||
public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { |
|||
return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); |
|||
} |
|||
|
|||
@Override |
|||
public List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception { |
|||
return listVersions(tenantId, branch, null, limit); |
|||
} |
|||
|
|||
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
checkRepository(); |
|||
checkBranch(tenantId, branch); |
|||
|
|||
return repository.listCommits(branch, path, limit).stream() |
|||
.map(this::toVersion) |
|||
.collect(Collectors.toList()); |
|||
|
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
|
|||
@Override |
|||
public List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() |
|||
.noneMatch(version -> version.getId().equals(versionId))) { |
|||
throw new IllegalArgumentException("Unknown version"); |
|||
} |
|||
return repository.listFilesAtCommit(versionId); |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
|
|||
@Override |
|||
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() |
|||
.noneMatch(version -> version.getId().equals(versionId))) { |
|||
throw new IllegalArgumentException("Unknown version"); |
|||
} |
|||
|
|||
String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); |
|||
return parseEntityData(entityDataJson); |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
@Override |
|||
public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { |
|||
EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); |
|||
return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() |
|||
.importInboundRelations(false) |
|||
.importOutboundRelations(false) |
|||
.updateReferencesToOtherEntities(true) |
|||
.build()); |
|||
} |
|||
|
|||
@Override |
|||
public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() |
|||
.map(entityDataFilePath -> { |
|||
String entityDataJson; |
|||
try { |
|||
entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); |
|||
} catch (IOException e) { |
|||
throw new UncheckedIOException(e); |
|||
} |
|||
return parseEntityData(entityDataJson); |
|||
}) |
|||
.collect(Collectors.toList()); |
|||
|
|||
return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() |
|||
.importInboundRelations(false) |
|||
.importOutboundRelations(false) |
|||
.updateReferencesToOtherEntities(true) |
|||
.build()); |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
private void tryFetch() throws GitAPIException { |
|||
repositoryLock.readLock().lock(); |
|||
try { |
|||
if (fetchLock.tryLock()) { |
|||
try { |
|||
log.info("Fetching remote repository"); |
|||
repository.fetch(); |
|||
} finally { |
|||
fetchLock.unlock(); |
|||
} |
|||
} |
|||
} finally { |
|||
repositoryLock.readLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
|
|||
private String getRelativePathForEntity(EntityId entityId) { |
|||
return getRelativePathForEntityType(entityId.getEntityType()) |
|||
+ "/" + entityId.getId() + ".json"; |
|||
} |
|||
|
|||
private String getRelativePathForEntityType(EntityType entityType) { |
|||
return entityType.name().toLowerCase(); |
|||
} |
|||
|
|||
|
|||
private void checkBranch(TenantId tenantId, String branch) { |
|||
// TODO [viacheslav]: all branches are available by default?
|
|||
if (!getAllowedBranches(tenantId).contains(branch)) { |
|||
throw new IllegalArgumentException("Tenant does not have access to this branch"); |
|||
} |
|||
} |
|||
|
|||
public Set<String> getAllowedBranches(TenantId tenantId) { |
|||
return Optional.ofNullable(getSettings()) |
|||
.flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) |
|||
.flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) |
|||
.orElse(Collections.emptySet()); |
|||
} |
|||
|
|||
private EntityVersion toVersion(Commit commit) { |
|||
return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); |
|||
} |
|||
|
|||
private <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> parseEntityData(String entityDataJson) { |
|||
return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {}); |
|||
} |
|||
|
|||
|
|||
|
|||
@Override |
|||
public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { |
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
this.repository = initRepository(settings.getGitSettings()); |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
|
|||
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) |
|||
.orElseGet(() -> { |
|||
AdminSettings newSettings = new AdminSettings(); |
|||
newSettings.setKey(SETTINGS_KEY); |
|||
return newSettings; |
|||
}); |
|||
adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); |
|||
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); |
|||
} |
|||
|
|||
@Override |
|||
public EntitiesVersionControlSettings getSettings() { |
|||
return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) |
|||
.map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) |
|||
.orElse(null); |
|||
} |
|||
|
|||
|
|||
private void checkRepository() { |
|||
if (repository == null) { |
|||
throw new IllegalStateException("Repository is not initialized"); |
|||
} |
|||
} |
|||
|
|||
private static Repository initRepository(GitSettings gitSettings) throws Exception { |
|||
if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { |
|||
return Repository.open(gitSettings.getRepositoryDirectory(), |
|||
gitSettings.getUsername(), gitSettings.getPassword()); |
|||
} else { |
|||
Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); |
|||
return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), |
|||
gitSettings.getUsername(), gitSettings.getPassword()); |
|||
} |
|||
} |
|||
|
|||
public void resetRepository() throws Exception { |
|||
repositoryLock.writeLock().lock(); |
|||
try { |
|||
if (this.repository != null) { |
|||
FileUtils.deleteDirectory(new File(repository.getDirectory())); |
|||
this.repository = null; |
|||
} |
|||
EntitiesVersionControlSettings settings = getSettings(); |
|||
this.repository = initRepository(settings.getGitSettings()); |
|||
} finally { |
|||
repositoryLock.writeLock().unlock(); |
|||
} |
|||
} |
|||
|
|||
} |
|||
@ -1,57 +0,0 @@ |
|||
/** |
|||
* Copyright © 2016-2022 The Thingsboard Authors |
|||
* |
|||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|||
* you may not use this file except in compliance with the License. |
|||
* You may obtain a copy of the License at |
|||
* |
|||
* http://www.apache.org/licenses/LICENSE-2.0
|
|||
* |
|||
* Unless required by applicable law or agreed to in writing, software |
|||
* distributed under the License is distributed on an "AS IS" BASIS, |
|||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|||
* See the License for the specific language governing permissions and |
|||
* limitations under the License. |
|||
*/ |
|||
package org.thingsboard.server.service.sync.vcs; |
|||
|
|||
import org.thingsboard.server.common.data.EntityType; |
|||
import org.thingsboard.server.common.data.ExportableEntity; |
|||
import org.thingsboard.server.common.data.id.EntityId; |
|||
import org.thingsboard.server.common.data.id.TenantId; |
|||
import org.thingsboard.server.service.sync.exporting.data.EntityExportData; |
|||
import org.thingsboard.server.service.sync.importing.EntityImportResult; |
|||
import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; |
|||
import org.thingsboard.server.service.sync.vcs.data.EntityVersion; |
|||
|
|||
import java.util.List; |
|||
|
|||
public interface EntitiesVersionControlService { |
|||
|
|||
EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; |
|||
|
|||
EntityVersion saveEntitiesVersion(TenantId tenantId, List<EntityId> entitiesIds, String branch, String versionName) throws Exception; |
|||
|
|||
|
|||
List<EntityVersion> listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; |
|||
|
|||
List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; |
|||
|
|||
List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception; |
|||
|
|||
|
|||
List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; |
|||
|
|||
|
|||
<E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; |
|||
|
|||
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; |
|||
|
|||
List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; |
|||
|
|||
|
|||
void saveSettings(EntitiesVersionControlSettings settings) throws Exception; |
|||
|
|||
EntitiesVersionControlSettings getSettings(); |
|||
|
|||
} |
|||
Loading…
Reference in new issue