Viacheslav Klimov
4 years ago
12 changed files with 665 additions and 632 deletions
@ -0,0 +1,395 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2022 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.sync.vc; |
||||
|
|
||||
|
import com.fasterxml.jackson.databind.ObjectMapper; |
||||
|
import com.fasterxml.jackson.databind.ObjectWriter; |
||||
|
import com.fasterxml.jackson.databind.SerializationFeature; |
||||
|
import lombok.RequiredArgsConstructor; |
||||
|
import lombok.SneakyThrows; |
||||
|
import lombok.extern.slf4j.Slf4j; |
||||
|
import org.apache.commons.io.FileUtils; |
||||
|
import org.apache.commons.lang3.StringUtils; |
||||
|
import org.eclipse.jgit.api.errors.GitAPIException; |
||||
|
import org.springframework.stereotype.Service; |
||||
|
import org.thingsboard.common.util.JacksonUtil; |
||||
|
import org.thingsboard.server.common.data.AdminSettings; |
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.ExportableEntity; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.EntityIdFactory; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.dao.settings.AdminSettingsService; |
||||
|
import org.thingsboard.server.queue.util.AfterStartUp; |
||||
|
import org.thingsboard.server.queue.util.TbCoreComponent; |
||||
|
import org.thingsboard.server.service.security.model.SecurityUser; |
||||
|
import org.thingsboard.server.service.sync.EntitiesExportImportService; |
||||
|
import org.thingsboard.server.service.sync.exporting.data.EntityExportData; |
||||
|
import org.thingsboard.server.service.sync.exporting.data.request.EntityExportSettings; |
||||
|
import org.thingsboard.server.service.sync.importing.data.EntityImportResult; |
||||
|
import org.thingsboard.server.service.sync.importing.data.EntityImportSettings; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; |
||||
|
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersion; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; |
||||
|
import org.thingsboard.server.utils.GitRepository; |
||||
|
|
||||
|
import java.io.File; |
||||
|
import java.io.IOException; |
||||
|
import java.nio.charset.StandardCharsets; |
||||
|
import java.nio.file.Files; |
||||
|
import java.nio.file.Path; |
||||
|
import java.util.ArrayList; |
||||
|
import java.util.Collections; |
||||
|
import java.util.List; |
||||
|
import java.util.Optional; |
||||
|
import java.util.concurrent.Executors; |
||||
|
import java.util.concurrent.ScheduledExecutorService; |
||||
|
import java.util.concurrent.ScheduledFuture; |
||||
|
import java.util.concurrent.TimeUnit; |
||||
|
import java.util.concurrent.locks.ReadWriteLock; |
||||
|
import java.util.concurrent.locks.ReentrantReadWriteLock; |
||||
|
import java.util.stream.Collectors; |
||||
|
|
||||
|
@Service |
||||
|
@TbCoreComponent |
||||
|
@RequiredArgsConstructor |
||||
|
@Slf4j |
||||
|
public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { |
||||
|
|
||||
|
private final EntitiesExportImportService exportImportService; |
||||
|
|
||||
|
private GitRepository repository; |
||||
|
private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); |
||||
|
|
||||
|
private ScheduledExecutorService fetchExecutor; |
||||
|
private ScheduledFuture<?> fetchTask; |
||||
|
|
||||
|
private final AdminSettingsService adminSettingsService; |
||||
|
private static final String SETTINGS_KEY = "vc"; |
||||
|
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); |
||||
|
|
||||
|
|
||||
|
@AfterStartUp |
||||
|
public void init() { |
||||
|
EntitiesVersionControlSettings settings = getSettings(); |
||||
|
if (settings != null) { |
||||
|
try { |
||||
|
initRepository(settings); |
||||
|
} catch (Exception e) { |
||||
|
log.debug("Failed to init repository", e); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
int fetchPeriod = settings == null || settings.getFetchPeriod() == 0 ? 10 : settings.getFetchPeriod(); |
||||
|
fetchExecutor = Executors.newSingleThreadScheduledExecutor(); |
||||
|
fetchTask = scheduleFetch(fetchPeriod); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { |
||||
|
return saveEntitiesVersion(user, List.of(entityId), branch, versionName, settings); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public EntityVersion saveEntitiesVersion(SecurityUser user, List<EntityId> entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception { |
||||
|
repositoryLock.writeLock().lock(); |
||||
|
try { |
||||
|
checkRepository(); |
||||
|
checkBranch(user.getTenantId(), branch); |
||||
|
|
||||
|
List<EntityExportData<?>> entityDataList = new ArrayList<>(); |
||||
|
EntityExportSettings exportSettings = EntityExportSettings.builder() |
||||
|
.exportRelations(settings.isSaveRelations()) |
||||
|
.build(); |
||||
|
for (EntityId entityId : entitiesIds) { |
||||
|
EntityExportData<ExportableEntity<EntityId>> entityData = exportImportService.exportEntity(user, entityId, exportSettings); |
||||
|
entityDataList.add(entityData); |
||||
|
} |
||||
|
|
||||
|
fetch(); |
||||
|
if (repository.listBranches().contains(branch)) { |
||||
|
repository.checkout(branch); |
||||
|
repository.merge(branch); |
||||
|
} else { |
||||
|
repository.createAndCheckoutOrphanBranch(branch); |
||||
|
} |
||||
|
|
||||
|
for (EntityExportData<?> entityData : entityDataList) { |
||||
|
String entityDataJson = jsonWriter.writeValueAsString(entityData); |
||||
|
FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePath(entityData.getEntityType(), |
||||
|
entityData.getEntity().getId().toString())), entityDataJson, StandardCharsets.UTF_8); |
||||
|
} |
||||
|
|
||||
|
GitRepository.Commit commit = repository.commit(versionName, "."); |
||||
|
repository.push(); |
||||
|
return toVersion(commit); |
||||
|
} finally { |
||||
|
repositoryLock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersion> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { |
||||
|
return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { |
||||
|
return listVersions(tenantId, branch, getRelativePath(entityType, null)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception { |
||||
|
return listVersions(tenantId, branch, null); |
||||
|
} |
||||
|
|
||||
|
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path) throws Exception { |
||||
|
repositoryLock.readLock().lock(); |
||||
|
try { |
||||
|
checkRepository(); |
||||
|
checkBranch(tenantId, branch); |
||||
|
|
||||
|
return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() |
||||
|
.map(this::toVersion) |
||||
|
.collect(Collectors.toList()); |
||||
|
} finally { |
||||
|
repositoryLock.readLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { |
||||
|
return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { |
||||
|
return listEntitiesAtVersion(tenantId, branch, versionId, null); |
||||
|
} |
||||
|
|
||||
|
private List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { |
||||
|
repositoryLock.readLock().lock(); |
||||
|
try { |
||||
|
checkRepository(); |
||||
|
checkBranch(tenantId, branch); |
||||
|
checkVersion(tenantId, branch, versionId, path); |
||||
|
|
||||
|
return repository.listFilesAtCommit(versionId, path).stream() |
||||
|
.map(filePath -> { |
||||
|
EntityId entityId = fromRelativePath(filePath); |
||||
|
EntityExportData<?> entityData = getEntityDataAtVersion(entityId, versionId); |
||||
|
|
||||
|
VersionedEntityInfo info = new VersionedEntityInfo(); |
||||
|
info.setExternalId(entityId); |
||||
|
info.setEntityName(entityData.getEntity().getName()); |
||||
|
return info; |
||||
|
}) |
||||
|
.collect(Collectors.toList()); |
||||
|
} finally { |
||||
|
repositoryLock.readLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
@Override |
||||
|
public EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { |
||||
|
return loadAtVersion(user, branch, versionId, getRelativePath(externalId.getEntityType(), externalId.getId().toString()), settings).get(0); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersionLoadResult> loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { |
||||
|
return loadAtVersion(user, branch, versionId, getRelativePath(entityType, null), settings); |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<EntityVersionLoadResult> loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception { |
||||
|
return loadAtVersion(user, branch, versionId, null, settings); |
||||
|
} |
||||
|
|
||||
|
private List<EntityVersionLoadResult> loadAtVersion(SecurityUser user, String branch, String versionId, String path, EntityVersionLoadSettings settings) throws Exception { |
||||
|
List<EntityExportData<?>> entityDataList = new ArrayList<>(); |
||||
|
repositoryLock.readLock().lock(); |
||||
|
try { |
||||
|
for (VersionedEntityInfo info : listEntitiesAtVersion(user.getTenantId(), branch, versionId, path)) { |
||||
|
EntityExportData<?> entityData = getEntityDataAtVersion(info.getExternalId(), versionId); |
||||
|
entityDataList.add(entityData); |
||||
|
} |
||||
|
} finally { |
||||
|
repositoryLock.readLock().unlock(); |
||||
|
} |
||||
|
|
||||
|
EntityImportSettings importSettings = EntityImportSettings.builder() |
||||
|
.updateRelations(settings.isLoadRelations()) |
||||
|
.findExistingByName(settings.isFindExistingEntityByName()) |
||||
|
.build(); |
||||
|
List<EntityImportResult<?>> importResults = exportImportService.importEntities(user, entityDataList, importSettings); |
||||
|
|
||||
|
return importResults.stream() |
||||
|
.map(importResult -> EntityVersionLoadResult.builder() |
||||
|
.previousEntityVersion(importResult.getOldEntity()) |
||||
|
.newEntityVersion(importResult.getSavedEntity()) |
||||
|
.entityType(importResult.getEntityType()) |
||||
|
.build()) |
||||
|
.collect(Collectors.toList()); |
||||
|
} |
||||
|
|
||||
|
@SneakyThrows |
||||
|
private EntityExportData<?> getEntityDataAtVersion(EntityId externalId, String versionId) { |
||||
|
repositoryLock.readLock().lock(); |
||||
|
try { |
||||
|
String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); |
||||
|
return JacksonUtil.fromString(entityDataJson, EntityExportData.class); |
||||
|
} finally { |
||||
|
repositoryLock.readLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private void fetch() throws GitAPIException { |
||||
|
repositoryLock.writeLock().lock(); |
||||
|
try { |
||||
|
repository.fetch(); |
||||
|
} finally { |
||||
|
repositoryLock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private ScheduledFuture<?> scheduleFetch(int fetchPeriod) { |
||||
|
return fetchExecutor.scheduleWithFixedDelay(() -> { |
||||
|
if (repository == null) return; |
||||
|
try { |
||||
|
fetch(); |
||||
|
} catch (Exception e) { |
||||
|
log.error("Failed to fetch remote repository", e); |
||||
|
} |
||||
|
}, fetchPeriod, fetchPeriod, TimeUnit.SECONDS); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private void checkVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { |
||||
|
if (listVersions(tenantId, branch, path).stream().noneMatch(version -> version.getId().equals(versionId))) { |
||||
|
throw new IllegalArgumentException("Version not found"); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public List<String> listAllowedBranches(TenantId tenantId) { |
||||
|
return Optional.ofNullable(getSettings()) |
||||
|
.flatMap(settings -> Optional.ofNullable(settings.getTenantsAllowedBranches())) |
||||
|
.flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) |
||||
|
.orElse(Collections.emptyList()); |
||||
|
} |
||||
|
|
||||
|
private void checkBranch(TenantId tenantId, String branch) { |
||||
|
if (!listAllowedBranches(tenantId).contains(branch)) { |
||||
|
throw new IllegalArgumentException("Tenant does not have access to the branch"); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private void checkRepository() { |
||||
|
if (repository == null) { |
||||
|
throw new IllegalStateException("Repository is not initialized"); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void initRepository(EntitiesVersionControlSettings settings) throws Exception { |
||||
|
repositoryLock.writeLock().lock(); |
||||
|
try { |
||||
|
if (Files.exists(Path.of(settings.getRepositoryDirectory()))) { |
||||
|
this.repository = GitRepository.open(settings.getRepositoryDirectory(), settings.getUsername(), settings.getPassword()); |
||||
|
} else { |
||||
|
Files.createDirectories(Path.of(settings.getRepositoryDirectory())); |
||||
|
this.repository = GitRepository.clone(settings.getRepositoryUri(), settings.getRepositoryDirectory(), |
||||
|
settings.getUsername(), settings.getPassword()); |
||||
|
} |
||||
|
} finally { |
||||
|
repositoryLock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
private void clearRepository() throws IOException { |
||||
|
repositoryLock.writeLock().lock(); |
||||
|
try { |
||||
|
if (repository != null) { |
||||
|
FileUtils.deleteDirectory(new File(repository.getDirectory())); |
||||
|
repository = null; |
||||
|
} |
||||
|
} finally { |
||||
|
repositoryLock.writeLock().unlock(); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
|
||||
|
@SneakyThrows |
||||
|
@Override |
||||
|
public void saveSettings(EntitiesVersionControlSettings settings) { |
||||
|
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) |
||||
|
.orElseGet(() -> { |
||||
|
AdminSettings newAdminSettings = new AdminSettings(); |
||||
|
newAdminSettings.setKey(SETTINGS_KEY); |
||||
|
return newAdminSettings; |
||||
|
}); |
||||
|
adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); |
||||
|
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); |
||||
|
|
||||
|
repositoryLock.writeLock().lock(); |
||||
|
try { |
||||
|
clearRepository(); |
||||
|
initRepository(settings); |
||||
|
} finally { |
||||
|
repositoryLock.writeLock().unlock(); |
||||
|
} |
||||
|
|
||||
|
if (settings.getFetchPeriod() != 0) { |
||||
|
fetchTask.cancel(true); |
||||
|
fetchTask = scheduleFetch(settings.getFetchPeriod()); |
||||
|
} |
||||
|
} |
||||
|
|
||||
|
@Override |
||||
|
public EntitiesVersionControlSettings getSettings() { |
||||
|
return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, "vc")) |
||||
|
.map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) |
||||
|
.orElse(null); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
private EntityVersion toVersion(GitRepository.Commit commit) { |
||||
|
return new EntityVersion(commit.getId(), commit.getMessage()); |
||||
|
} |
||||
|
|
||||
|
private String getRelativePath(EntityType entityType, String entityId) { |
||||
|
String path = entityType.name().toLowerCase(); |
||||
|
if (entityId != null) { |
||||
|
path += "/" + entityId + ".json"; |
||||
|
} |
||||
|
return path; |
||||
|
} |
||||
|
|
||||
|
private EntityId fromRelativePath(String path) { |
||||
|
EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/")); |
||||
|
String entityId = StringUtils.substringBetween(path, "/", ".json"); |
||||
|
return EntityIdFactory.getByTypeAndUuid(entityType, entityId); |
||||
|
} |
||||
|
|
||||
|
} |
||||
@ -0,0 +1,64 @@ |
|||||
|
/** |
||||
|
* Copyright © 2016-2022 The Thingsboard Authors |
||||
|
* |
||||
|
* Licensed under the Apache License, Version 2.0 (the "License"); |
||||
|
* you may not use this file except in compliance with the License. |
||||
|
* You may obtain a copy of the License at |
||||
|
* |
||||
|
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
* |
||||
|
* Unless required by applicable law or agreed to in writing, software |
||||
|
* distributed under the License is distributed on an "AS IS" BASIS, |
||||
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
|
* See the License for the specific language governing permissions and |
||||
|
* limitations under the License. |
||||
|
*/ |
||||
|
package org.thingsboard.server.service.sync.vc; |
||||
|
|
||||
|
import org.thingsboard.server.common.data.EntityType; |
||||
|
import org.thingsboard.server.common.data.id.EntityId; |
||||
|
import org.thingsboard.server.common.data.id.TenantId; |
||||
|
import org.thingsboard.server.service.security.model.SecurityUser; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntitiesVersionControlSettings; |
||||
|
import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersion; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadResult; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersionLoadSettings; |
||||
|
import org.thingsboard.server.service.sync.vc.data.EntityVersionSaveSettings; |
||||
|
|
||||
|
import java.util.List; |
||||
|
|
||||
|
public interface EntitiesVersionControlService { |
||||
|
|
||||
|
EntityVersion saveEntityVersion(SecurityUser user, EntityId entityId, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; |
||||
|
|
||||
|
EntityVersion saveEntitiesVersion(SecurityUser user, List<EntityId> entitiesIds, String branch, String versionName, EntityVersionSaveSettings settings) throws Exception; |
||||
|
|
||||
|
|
||||
|
List<EntityVersion> listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception; |
||||
|
|
||||
|
List<EntityVersion> listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception; |
||||
|
|
||||
|
List<EntityVersion> listVersions(TenantId tenantId, String branch) throws Exception; |
||||
|
|
||||
|
|
||||
|
List<VersionedEntityInfo> listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; // will be good to return entity name also
|
||||
|
|
||||
|
List<VersionedEntityInfo> listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; |
||||
|
|
||||
|
|
||||
|
EntityVersionLoadResult loadEntityVersion(SecurityUser user, EntityId externalId, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; |
||||
|
|
||||
|
List<EntityVersionLoadResult> loadEntityTypeVersion(SecurityUser user, EntityType entityType, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; |
||||
|
|
||||
|
List<EntityVersionLoadResult> loadAllAtVersion(SecurityUser user, String branch, String versionId, EntityVersionLoadSettings settings) throws Exception; |
||||
|
|
||||
|
|
||||
|
List<String> listAllowedBranches(TenantId tenantId); |
||||
|
|
||||
|
|
||||
|
void saveSettings(EntitiesVersionControlSettings settings); |
||||
|
|
||||
|
EntitiesVersionControlSettings getSettings(); |
||||
|
|
||||
|
} |
||||
@ -1,358 +0,0 @@ |
|||||
/** |
|
||||
* Copyright © 2016-2022 The Thingsboard Authors |
|
||||
* |
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||
* you may not use this file except in compliance with the License. |
|
||||
* You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
package org.thingsboard.server.service.sync.vcs; |
|
||||
|
|
||||
import com.fasterxml.jackson.core.type.TypeReference; |
|
||||
import com.fasterxml.jackson.databind.ObjectMapper; |
|
||||
import com.fasterxml.jackson.databind.ObjectWriter; |
|
||||
import com.fasterxml.jackson.databind.SerializationFeature; |
|
||||
import lombok.RequiredArgsConstructor; |
|
||||
import lombok.extern.slf4j.Slf4j; |
|
||||
import org.apache.commons.io.FileUtils; |
|
||||
import org.eclipse.jgit.api.errors.GitAPIException; |
|
||||
import org.springframework.scheduling.annotation.Scheduled; |
|
||||
import org.springframework.stereotype.Service; |
|
||||
import org.thingsboard.common.util.JacksonUtil; |
|
||||
import org.thingsboard.server.common.data.AdminSettings; |
|
||||
import org.thingsboard.server.common.data.EntityType; |
|
||||
import org.thingsboard.server.common.data.ExportableEntity; |
|
||||
import org.thingsboard.server.common.data.id.EntityId; |
|
||||
import org.thingsboard.server.common.data.id.TenantId; |
|
||||
import org.thingsboard.server.dao.settings.AdminSettingsService; |
|
||||
import org.thingsboard.server.dao.tenant.TenantService; |
|
||||
import org.thingsboard.server.queue.util.AfterStartUp; |
|
||||
import org.thingsboard.server.queue.util.TbCoreComponent; |
|
||||
import org.thingsboard.server.service.sync.EntitiesExportImportService; |
|
||||
import org.thingsboard.server.service.sync.exporting.EntityExportSettings; |
|
||||
import org.thingsboard.server.service.sync.exporting.data.EntityExportData; |
|
||||
import org.thingsboard.server.service.sync.importing.EntityImportResult; |
|
||||
import org.thingsboard.server.service.sync.importing.EntityImportSettings; |
|
||||
import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; |
|
||||
import org.thingsboard.server.service.sync.vcs.data.EntityVersion; |
|
||||
import org.thingsboard.server.service.sync.vcs.data.GitSettings; |
|
||||
import org.thingsboard.server.utils.git.Repository; |
|
||||
import org.thingsboard.server.utils.git.data.Commit; |
|
||||
|
|
||||
import java.io.File; |
|
||||
import java.io.IOException; |
|
||||
import java.io.UncheckedIOException; |
|
||||
import java.nio.charset.StandardCharsets; |
|
||||
import java.nio.file.Files; |
|
||||
import java.nio.file.Path; |
|
||||
import java.util.Collections; |
|
||||
import java.util.List; |
|
||||
import java.util.Optional; |
|
||||
import java.util.Set; |
|
||||
import java.util.concurrent.locks.Lock; |
|
||||
import java.util.concurrent.locks.ReadWriteLock; |
|
||||
import java.util.concurrent.locks.ReentrantLock; |
|
||||
import java.util.concurrent.locks.ReentrantReadWriteLock; |
|
||||
import java.util.stream.Collectors; |
|
||||
|
|
||||
@Service |
|
||||
@TbCoreComponent |
|
||||
@RequiredArgsConstructor |
|
||||
@Slf4j |
|
||||
public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { |
|
||||
// TODO [viacheslav]: start up only on one of the cores
|
|
||||
|
|
||||
private final TenantService tenantService; |
|
||||
private final EntitiesExportImportService exportImportService; |
|
||||
private final AdminSettingsService adminSettingsService; |
|
||||
|
|
||||
private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); |
|
||||
private static final String SETTINGS_KEY = "vc"; |
|
||||
|
|
||||
private Repository repository; |
|
||||
private final Lock fetchLock = new ReentrantLock(); |
|
||||
private final ReadWriteLock repositoryLock = new ReentrantReadWriteLock(); |
|
||||
|
|
||||
@AfterStartUp |
|
||||
public void init() throws Exception { |
|
||||
try { |
|
||||
EntitiesVersionControlSettings settings = getSettings(); |
|
||||
if (settings != null && settings.getGitSettings() != null) { |
|
||||
this.repository = initRepository(settings.getGitSettings()); |
|
||||
} |
|
||||
} catch (Exception e) { |
|
||||
log.error("Failed to initialize entities version control service", e); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
@Scheduled(initialDelay = 10 * 1000, fixedDelay = 10 * 1000) |
|
||||
private void fetch() throws Exception { |
|
||||
if (repository == null) return; |
|
||||
tryFetch(); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
@Override |
|
||||
public EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception { |
|
||||
return saveEntitiesVersion(tenantId, List.of(entityId), branch, versionName); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public EntityVersion saveEntitiesVersion(TenantId tenantId, List<EntityId> entitiesIds, String branch, String versionName) throws Exception { |
|
||||
checkRepository(); |
|
||||
checkBranch(tenantId, branch); |
|
||||
|
|
||||
EntityExportSettings exportSettings = EntityExportSettings.builder() |
|
||||
.exportInboundRelations(false) |
|
||||
.exportOutboundRelations(false) |
|
||||
.build(); |
|
||||
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = entitiesIds.stream() |
|
||||
.map(entityId -> exportImportService.exportEntity(tenantId, entityId, exportSettings)) |
|
||||
.collect(Collectors.toList()); |
|
||||
|
|
||||
tryFetch(); |
|
||||
|
|
||||
repositoryLock.writeLock().lock(); |
|
||||
try { |
|
||||
if (repository.listBranches().contains(branch)) { |
|
||||
repository.checkout(branch); |
|
||||
repository.merge(branch); |
|
||||
} else { |
|
||||
repository.createAndCheckoutOrphanBranch(branch); |
|
||||
} |
|
||||
|
|
||||
for (EntityExportData<ExportableEntity<EntityId>> entityData : entityDataList) { |
|
||||
String entityDataJson = jsonWriter.writeValueAsString(entityData); |
|
||||
FileUtils.write(new File(repository.getDirectory() + "/" + getRelativePathForEntity(entityData.getEntity().getId())), |
|
||||
entityDataJson, StandardCharsets.UTF_8); |
|
||||
} |
|
||||
|
|
||||
Commit commit = repository.commit(versionName, ".", "Tenant " + tenantId); |
|
||||
repository.push(); |
|
||||
return toVersion(commit); |
|
||||
} finally { |
|
||||
repositoryLock.writeLock().unlock(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
|
|
||||
@Override |
|
||||
public List<EntityVersion> listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception { |
|
||||
return listVersions(tenantId, branch, getRelativePathForEntity(entityId), limit); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception { |
|
||||
return listVersions(tenantId, getRelativePathForEntityType(entityType), limit); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception { |
|
||||
return listVersions(tenantId, branch, null, limit); |
|
||||
} |
|
||||
|
|
||||
private List<EntityVersion> listVersions(TenantId tenantId, String branch, String path, int limit) throws Exception { |
|
||||
repositoryLock.readLock().lock(); |
|
||||
try { |
|
||||
checkRepository(); |
|
||||
checkBranch(tenantId, branch); |
|
||||
|
|
||||
return repository.listCommits(branch, path, limit).stream() |
|
||||
.map(this::toVersion) |
|
||||
.collect(Collectors.toList()); |
|
||||
|
|
||||
} finally { |
|
||||
repositoryLock.readLock().unlock(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
|
|
||||
@Override |
|
||||
public List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { |
|
||||
repositoryLock.readLock().lock(); |
|
||||
try { |
|
||||
if (listVersions(tenantId, branch, Integer.MAX_VALUE).stream() |
|
||||
.noneMatch(version -> version.getId().equals(versionId))) { |
|
||||
throw new IllegalArgumentException("Unknown version"); |
|
||||
} |
|
||||
return repository.listFilesAtCommit(versionId); |
|
||||
} finally { |
|
||||
repositoryLock.readLock().unlock(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
|
|
||||
@Override |
|
||||
public <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { |
|
||||
repositoryLock.readLock().lock(); |
|
||||
try { |
|
||||
if (listEntityVersions(tenantId, entityId, branch, Integer.MAX_VALUE).stream() |
|
||||
.noneMatch(version -> version.getId().equals(versionId))) { |
|
||||
throw new IllegalArgumentException("Unknown version"); |
|
||||
} |
|
||||
|
|
||||
String entityDataJson = repository.getFileContentAtCommit(getRelativePathForEntity(entityId), versionId); |
|
||||
return parseEntityData(entityDataJson); |
|
||||
} finally { |
|
||||
repositoryLock.readLock().unlock(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
@Override |
|
||||
public <E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception { |
|
||||
EntityExportData<E> entityData = getEntityAtVersion(tenantId, entityId, branch, versionId); |
|
||||
return exportImportService.importEntity(tenantId, entityData, EntityImportSettings.builder() |
|
||||
.importInboundRelations(false) |
|
||||
.importOutboundRelations(false) |
|
||||
.updateReferencesToOtherEntities(true) |
|
||||
.build()); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { |
|
||||
repositoryLock.readLock().lock(); |
|
||||
try { |
|
||||
List<EntityExportData<ExportableEntity<EntityId>>> entityDataList = listFilesAtVersion(tenantId, branch, versionId).stream() |
|
||||
.map(entityDataFilePath -> { |
|
||||
String entityDataJson; |
|
||||
try { |
|
||||
entityDataJson = repository.getFileContentAtCommit(entityDataFilePath, versionId); |
|
||||
} catch (IOException e) { |
|
||||
throw new UncheckedIOException(e); |
|
||||
} |
|
||||
return parseEntityData(entityDataJson); |
|
||||
}) |
|
||||
.collect(Collectors.toList()); |
|
||||
|
|
||||
return exportImportService.importEntities(tenantId, entityDataList, EntityImportSettings.builder() |
|
||||
.importInboundRelations(false) |
|
||||
.importOutboundRelations(false) |
|
||||
.updateReferencesToOtherEntities(true) |
|
||||
.build()); |
|
||||
} finally { |
|
||||
repositoryLock.readLock().unlock(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private void tryFetch() throws GitAPIException { |
|
||||
repositoryLock.readLock().lock(); |
|
||||
try { |
|
||||
if (fetchLock.tryLock()) { |
|
||||
try { |
|
||||
log.info("Fetching remote repository"); |
|
||||
repository.fetch(); |
|
||||
} finally { |
|
||||
fetchLock.unlock(); |
|
||||
} |
|
||||
} |
|
||||
} finally { |
|
||||
repositoryLock.readLock().unlock(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
|
|
||||
private String getRelativePathForEntity(EntityId entityId) { |
|
||||
return getRelativePathForEntityType(entityId.getEntityType()) |
|
||||
+ "/" + entityId.getId() + ".json"; |
|
||||
} |
|
||||
|
|
||||
private String getRelativePathForEntityType(EntityType entityType) { |
|
||||
return entityType.name().toLowerCase(); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
private void checkBranch(TenantId tenantId, String branch) { |
|
||||
// TODO [viacheslav]: all branches are available by default?
|
|
||||
if (!getAllowedBranches(tenantId).contains(branch)) { |
|
||||
throw new IllegalArgumentException("Tenant does not have access to this branch"); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public Set<String> getAllowedBranches(TenantId tenantId) { |
|
||||
return Optional.ofNullable(getSettings()) |
|
||||
.flatMap(settings -> Optional.ofNullable(settings.getAllowedBranches())) |
|
||||
.flatMap(tenantsAllowedBranches -> Optional.ofNullable(tenantsAllowedBranches.get(tenantId.getId()))) |
|
||||
.orElse(Collections.emptySet()); |
|
||||
} |
|
||||
|
|
||||
private EntityVersion toVersion(Commit commit) { |
|
||||
return new EntityVersion(commit.getId(), commit.getMessage(), commit.getAuthorName()); |
|
||||
} |
|
||||
|
|
||||
private <E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> parseEntityData(String entityDataJson) { |
|
||||
return JacksonUtil.fromString(entityDataJson, new TypeReference<EntityExportData<E>>() {}); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
|
|
||||
@Override |
|
||||
public void saveSettings(EntitiesVersionControlSettings settings) throws Exception { |
|
||||
repositoryLock.writeLock().lock(); |
|
||||
try { |
|
||||
this.repository = initRepository(settings.getGitSettings()); |
|
||||
} finally { |
|
||||
repositoryLock.writeLock().unlock(); |
|
||||
} |
|
||||
|
|
||||
AdminSettings adminSettings = Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) |
|
||||
.orElseGet(() -> { |
|
||||
AdminSettings newSettings = new AdminSettings(); |
|
||||
newSettings.setKey(SETTINGS_KEY); |
|
||||
return newSettings; |
|
||||
}); |
|
||||
adminSettings.setJsonValue(JacksonUtil.valueToTree(settings)); |
|
||||
adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminSettings); |
|
||||
} |
|
||||
|
|
||||
@Override |
|
||||
public EntitiesVersionControlSettings getSettings() { |
|
||||
return Optional.ofNullable(adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, SETTINGS_KEY)) |
|
||||
.map(adminSettings -> JacksonUtil.treeToValue(adminSettings.getJsonValue(), EntitiesVersionControlSettings.class)) |
|
||||
.orElse(null); |
|
||||
} |
|
||||
|
|
||||
|
|
||||
private void checkRepository() { |
|
||||
if (repository == null) { |
|
||||
throw new IllegalStateException("Repository is not initialized"); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
private static Repository initRepository(GitSettings gitSettings) throws Exception { |
|
||||
if (Files.exists(Path.of(gitSettings.getRepositoryDirectory()))) { |
|
||||
return Repository.open(gitSettings.getRepositoryDirectory(), |
|
||||
gitSettings.getUsername(), gitSettings.getPassword()); |
|
||||
} else { |
|
||||
Files.createDirectories(Path.of(gitSettings.getRepositoryDirectory())); |
|
||||
return Repository.clone(gitSettings.getRepositoryUri(), gitSettings.getRepositoryDirectory(), |
|
||||
gitSettings.getUsername(), gitSettings.getPassword()); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
public void resetRepository() throws Exception { |
|
||||
repositoryLock.writeLock().lock(); |
|
||||
try { |
|
||||
if (this.repository != null) { |
|
||||
FileUtils.deleteDirectory(new File(repository.getDirectory())); |
|
||||
this.repository = null; |
|
||||
} |
|
||||
EntitiesVersionControlSettings settings = getSettings(); |
|
||||
this.repository = initRepository(settings.getGitSettings()); |
|
||||
} finally { |
|
||||
repositoryLock.writeLock().unlock(); |
|
||||
} |
|
||||
} |
|
||||
|
|
||||
} |
|
||||
@ -1,57 +0,0 @@ |
|||||
/** |
|
||||
* Copyright © 2016-2022 The Thingsboard Authors |
|
||||
* |
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); |
|
||||
* you may not use this file except in compliance with the License. |
|
||||
* You may obtain a copy of the License at |
|
||||
* |
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||
* |
|
||||
* Unless required by applicable law or agreed to in writing, software |
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, |
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
|
||||
* See the License for the specific language governing permissions and |
|
||||
* limitations under the License. |
|
||||
*/ |
|
||||
package org.thingsboard.server.service.sync.vcs; |
|
||||
|
|
||||
import org.thingsboard.server.common.data.EntityType; |
|
||||
import org.thingsboard.server.common.data.ExportableEntity; |
|
||||
import org.thingsboard.server.common.data.id.EntityId; |
|
||||
import org.thingsboard.server.common.data.id.TenantId; |
|
||||
import org.thingsboard.server.service.sync.exporting.data.EntityExportData; |
|
||||
import org.thingsboard.server.service.sync.importing.EntityImportResult; |
|
||||
import org.thingsboard.server.service.sync.vcs.data.EntitiesVersionControlSettings; |
|
||||
import org.thingsboard.server.service.sync.vcs.data.EntityVersion; |
|
||||
|
|
||||
import java.util.List; |
|
||||
|
|
||||
public interface EntitiesVersionControlService { |
|
||||
|
|
||||
EntityVersion saveEntityVersion(TenantId tenantId, EntityId entityId, String branch, String versionName) throws Exception; |
|
||||
|
|
||||
EntityVersion saveEntitiesVersion(TenantId tenantId, List<EntityId> entitiesIds, String branch, String versionName) throws Exception; |
|
||||
|
|
||||
|
|
||||
List<EntityVersion> listEntityVersions(TenantId tenantId, EntityId entityId, String branch, int limit) throws Exception; |
|
||||
|
|
||||
List<EntityVersion> listEntityTypeVersions(TenantId tenantId, EntityType entityType, String branch, int limit) throws Exception; |
|
||||
|
|
||||
List<EntityVersion> listVersions(TenantId tenantId, String branch, int limit) throws Exception; |
|
||||
|
|
||||
|
|
||||
List<String> listFilesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; |
|
||||
|
|
||||
|
|
||||
<E extends ExportableEntity<I>, I extends EntityId> EntityExportData<E> getEntityAtVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; |
|
||||
|
|
||||
<E extends ExportableEntity<I>, I extends EntityId> EntityImportResult<E> loadEntityVersion(TenantId tenantId, I entityId, String branch, String versionId) throws Exception; |
|
||||
|
|
||||
List<EntityImportResult<ExportableEntity<EntityId>>> loadAllAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; |
|
||||
|
|
||||
|
|
||||
void saveSettings(EntitiesVersionControlSettings settings) throws Exception; |
|
||||
|
|
||||
EntitiesVersionControlSettings getSettings(); |
|
||||
|
|
||||
} |
|
||||
Loading…
Reference in new issue