diff --git a/application/pom.xml b/application/pom.xml index 83d36c2b42..37d782b3b0 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -69,6 +69,10 @@ org.thingsboard.common cluster-api + + org.thingsboard.common + version-control + org.thingsboard.rule-engine rule-engine-components @@ -341,14 +345,6 @@ Java-WebSocket test - - org.eclipse.jgit - org.eclipse.jgit - - - org.eclipse.jgit - org.eclipse.jgit.ssh.apache - diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 3868476491..237f681819 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -29,7 +29,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; -import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/AssetController.java b/application/src/main/java/org/thingsboard/server/controller/AssetController.java index 01401deebe..2a3268146f 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AssetController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AssetController.java @@ -51,8 +51,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.asset.AssetBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java index 61e44241d7..8b6f49c491 100644 --- a/application/src/main/java/org/thingsboard/server/controller/DeviceController.java +++ b/application/src/main/java/org/thingsboard/server/controller/DeviceController.java @@ -66,8 +66,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.device.DeviceBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; diff --git a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java index 8136b6d052..b3bbb03f4c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EdgeController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EdgeController.java @@ -52,8 +52,8 @@ import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeBulkImportService; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportRequest; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportResult; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportRequest; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportResult; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index 599149402f..bba5d20b06 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -27,12 +27,12 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; import java.util.ArrayList; import java.util.List; @@ -65,43 +65,27 @@ public class EntitiesVersionControlController extends BaseController { " \"saveRelations\": true\n" + " }\n" + "}\n```" + NEW_LINE + - "ENTITY_LIST:" + NEW_LINE + + "COMPLEX:" + NEW_LINE + "```\n{\n" + - " \"type\": \"ENTITY_LIST\",\n" + + " \"type\": \"COMPLEX\",\n" + "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + - "\n" + - " \"entitiesIds\": [\n" + - " {\n" + - " \"entityType\": \"DEVICE\",\n" + - " \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + - " },\n" + - " {\n" + - " \"entityType\": \"DEVICE_PROFILE\",\n" + - " \"id\": \"b7944123-d4f4-11ec-847b-0f432358ab48\"\n" + - " }\n" + - " ],\n" + - " \"config\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + - " }\n" + - "}\n```" + NEW_LINE + - "ENTITY_TYPE:" + NEW_LINE + - "```\n{\n" + - " \"type\": \"ENTITY_TYPE\",\n" + - "\n" + - " \"versionName\": \"Version 1.0\",\n" + - " \"branch\": \"dev\",\n" + + " \"versionName\": \"Devices and profiles: release 2\",\n" + + " \"branch\": \"master\",\n" + "\n" + + " \"syncStrategy\": \"OVERWRITE\",\n" + " \"entityTypes\": {\n" + " \"DEVICE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"MERGE\"\n" + + " \"syncStrategy\": null,\n" + + " \"allEntities\": true,\n" + + " \"saveRelations\": true\n" + " },\n" + " \"DEVICE_PROFILE\": {\n" + - " \"saveRelations\": true,\n" + - " \"syncStrategy\": \"OVERWRITE\"\n" + + " \"syncStrategy\": \"MERGE\",\n" + + " \"allEntities\": false,\n" + + " \"entityIds\": [\n" + + " \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" + + " ],\n" + + " \"saveRelations\": true\n" + " }\n" + " }\n" + "}\n```") @@ -182,7 +166,7 @@ public class EntitiesVersionControlController extends BaseController { @PathVariable EntityType entityType, @PathVariable String versionId) throws ThingsboardException { try { - return versionControlService.listEntitiesAtVersion(getTenantId(), entityType, branch, versionId); + return versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType); } catch (Exception e) { throw handleException(e); } diff --git a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java index a2b71267b4..5eaeec0b73 100644 --- a/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/asset/AssetBulkImportService.java @@ -26,8 +26,8 @@ import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.asset.TbAssetService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java index ee119c4dce..f0a7b6231e 100644 --- a/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/device/DeviceBulkImportService.java @@ -49,8 +49,8 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.device.TbDeviceService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java index fea50538cc..b0fb2161e9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeBulkImportService.java @@ -28,8 +28,8 @@ import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.edge.EdgeService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.importing.csv.AbstractBulkImportService; -import org.thingsboard.server.service.sync.exportimport.importing.csv.BulkImportColumnType; +import org.thingsboard.server.service.sync.ie.importing.csv.AbstractBulkImportService; +import org.thingsboard.server.service.sync.ie.importing.csv.BulkImportColumnType; import org.thingsboard.server.service.entitiy.edge.TbEdgeService; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 108c2c0b1b..38d3b00213 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -27,15 +27,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.BaseEntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.impl.DefaultEntityExportService; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; import java.util.ArrayList; import java.util.Collection; @@ -95,7 +95,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Transactional(rollbackFor = Exception.class, timeout = 120) @Override public List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException { - fixDataOrderForImport(exportDataList); + exportDataList.sort(getDataComparatorForImport()); List> importResults = new ArrayList<>(); @@ -125,9 +125,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importResults; } + + @Override + public Comparator> getDataComparatorForImport() { + return Comparator.comparing(EntityExportData::getEntityType, getEntityTypeComparatorForImport()); + } + @Override - public void fixDataOrderForImport(List> exportDataList) { - exportDataList.sort(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))); + public Comparator getEntityTypeComparatorForImport() { + return Comparator.comparing(SUPPORTED_ENTITY_TYPES::indexOf); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java similarity index 75% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 049cf4390a..7bdb271444 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -13,17 +13,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport; +package org.thingsboard.server.service.sync.ie; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import java.util.Comparator; import java.util.List; public interface EntitiesExportImportService { @@ -35,6 +37,9 @@ public interface EntitiesExportImportService { List> importEntities(SecurityUser user, List> exportDataList, EntityImportSettings importSettings) throws ThingsboardException; - void fixDataOrderForImport(List> exportDataList); + + Comparator> getDataComparatorForImport(); + + Comparator getEntityTypeComparatorForImport(); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java index dafd68ba7b..18c5f5c7c3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/DefaultExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/DefaultExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java similarity index 81% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 5532f30125..9b251a3dab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -13,14 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityExportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java index 66c9079148..742b993cfc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/ExportableEntitiesService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/ExportableEntitiesService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting; +package org.thingsboard.server.service.sync.ie.exporting; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -25,9 +25,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateConfig; - -import java.util.List; public interface ExportableEntitiesService { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java similarity index 86% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 46428a763f..80850cb462 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; @@ -21,8 +21,8 @@ import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 69af613480..31bf768e90 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; @@ -28,10 +28,10 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.EntityExportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java index 93f9c51655..2e03b92a44 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/DeviceExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DeviceExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java index f40ee6884a..15c9c72a74 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/impl/RuleChainExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/RuleChainExportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.impl; +package org.thingsboard.server.service.sync.ie.exporting.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -23,7 +23,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import java.util.Set; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java similarity index 78% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java index be6f62efe4..bf249937b9 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/EntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/EntityImportService.java @@ -13,16 +13,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing; +package org.thingsboard.server.service.sync.ie.importing; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; public interface EntityImportService, D extends EntityExportData> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java similarity index 99% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java index 2be56ebe2a..b92e100401 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/AbstractBulkImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/AbstractBulkImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import com.google.common.util.concurrent.FutureCallback; import com.google.gson.JsonObject; @@ -66,7 +66,6 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java index 9c0e6b6491..24b566e631 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportColumnType.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportColumnType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Getter; import org.thingsboard.server.common.data.DataConstants; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java index 6347910cab..e8eac6a9ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportRequest.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java index 550d4a72f3..0626c8e690 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/BulkImportResult.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/BulkImportResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java index a39dcc7481..d48e9a3d23 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/csv/ImportedEntityInfo.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/csv/ImportedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.csv; +package org.thingsboard.server.service.sync.ie.importing.csv; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index 80a942c73f..fe6c3f1875 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7ecfa7d3ab..203b0caa16 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Autowired; @@ -35,11 +35,11 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.importing.EntityImportService; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.ie.importing.EntityImportService; import java.util.ArrayList; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java index b69c5a0d5b..2a7570e644 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/CustomerImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -26,7 +26,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index b85e403f87..16ebdd580e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -30,8 +30,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index 112152365b..eb123a63af 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.DeviceExportData; +import org.thingsboard.server.common.data.sync.ie.DeviceExportData; @Service @TbCoreComponent diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java index f97950aaf8..592fe4bc5e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/DeviceProfileImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; @@ -28,7 +28,7 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.ota.OtaPackageStateService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.Objects; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java rename to application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 993f9fa27a..1f1f15a74e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.impl; +package org.thingsboard.server.service.sync.ie.importing.impl; import com.fasterxml.jackson.databind.JsonNode; import lombok.RequiredArgsConstructor; @@ -32,8 +32,8 @@ import org.thingsboard.server.common.data.rule.RuleChainUpdateResult; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.exportimport.exporting.data.RuleChainExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.ie.RuleChainExportData; import org.thingsboard.server.utils.RegexUtils; import java.util.Collections; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 77a5586a8c..1496882031 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -15,14 +15,9 @@ */ package org.thingsboard.server.service.sync.vc; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.springframework.beans.factory.annotation.Value; +import org.apache.commons.lang3.ObjectUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; @@ -32,270 +27,157 @@ import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.query.*; -import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.*; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadConfig; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.settings.AdminSettingsService; -import org.thingsboard.server.dao.tenant.TenantDao; -import org.thingsboard.server.queue.util.AfterStartUp; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.service.sync.exportimport.EntitiesExportImportService; -import org.thingsboard.server.service.sync.exportimport.exporting.ExportableEntitiesService; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportData; -import org.thingsboard.server.service.sync.exportimport.exporting.data.EntityExportSettings; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportResult; -import org.thingsboard.server.service.sync.exportimport.importing.data.EntityImportSettings; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.*; -import org.thingsboard.server.service.sync.vc.data.request.load.EntityTypeVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.SingleEntityVersionLoadRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadConfig; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; -import org.thingsboard.server.utils.GitRepository; -import org.thingsboard.server.utils.ThrowingRunnable; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; +import org.thingsboard.server.service.sync.ie.EntitiesExportImportService; +import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.SyncStrategy; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.EntityTypeVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; -import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; - @Service @TbCoreComponent @RequiredArgsConstructor @Slf4j public class DefaultEntitiesVersionControlService implements EntitiesVersionControlService { + private final GitVersionControlService gitService; private final EntitiesExportImportService exportImportService; private final ExportableEntitiesService exportableEntitiesService; private final AdminSettingsService adminSettingsService; private final EntityService entityService; - private final TenantDao tenantDao; private final TransactionTemplate transactionTemplate; - // TODO [viacheslav]: concurrency - private final Map repositories = new ConcurrentHashMap<>(); - @Value("${java.io.tmpdir}/repositories") - private String repositoriesFolder; - - private static final String SETTINGS_KEY = "entitiesVersionControl"; - private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); - - - @AfterStartUp - public void init() { - DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { - EntitiesVersionControlSettings settings = getVersionControlSettings(tenantId); - if (settings != null) { - try { - initRepository(tenantId, settings); - } catch (Exception e) { - log.warn("Failed to init repository for tenant {}", tenantId, e); - } - } - }); - Executors.newSingleThreadScheduledExecutor().scheduleWithFixedDelay(() -> { - repositories.forEach((tenantId, repository) -> { - try { - repository.fetch(); - log.info("Fetching remote repository for tenant {}", tenantId); - } catch (Exception e) { - log.warn("Failed to fetch repository for tenant {}", tenantId, e); - } - }); - }, 5, 5, TimeUnit.SECONDS); - } - + public static final String SETTINGS_KEY = "entitiesVersionControl"; @Override public VersionCreationResult saveEntitiesVersion(SecurityUser user, VersionCreateRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - repository.fetch(); - if (repository.listBranches().contains(request.getBranch())) { - repository.checkout(request.getBranch()); - repository.merge(request.getBranch()); - } else { // TODO [viacheslav]: rollback orphan branch on failure - repository.createAndCheckoutOrphanBranch(request.getBranch()); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch - } + var commit = gitService.prepareCommit(user.getTenantId(), request); switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionCreateRequest versionCreateRequest = (SingleEntityVersionCreateRequest) request; - saveEntityData(user, repository, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); - break; - } - case ENTITY_LIST: { - EntityListVersionCreateRequest versionCreateRequest = (EntityListVersionCreateRequest) request; - if (versionCreateRequest.getConfig().getSyncStrategy() == SyncStrategy.OVERWRITE) { - versionCreateRequest.getEntitiesIds().stream() - .map(EntityId::getEntityType).distinct() - .forEach(entityType -> { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - for (EntityId entityId : versionCreateRequest.getEntitiesIds()) { - saveEntityData(user, repository, entityId, versionCreateRequest.getConfig()); - } + saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig()); break; } case COMPLEX: { ComplexVersionCreateRequest versionCreateRequest = (ComplexVersionCreateRequest) request; versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (config.getSyncStrategy() == SyncStrategy.OVERWRITE) { - try { - FileUtils.deleteDirectory(Path.of(repository.getDirectory(), getRelativePath(entityType, null)).toFile()); - } catch (IOException e) { - throw new RuntimeException(e); - } + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { + gitService.deleteAll(commit, entityType); } - EntityTypeFilter entityTypeFilter = new EntityTypeFilter(); - entityTypeFilter.setEntityType(entityType); - EntityDataPageLink entityDataPageLink = new EntityDataPageLink(); - entityDataPageLink.setPage(-1); - entityDataPageLink.setPageSize(-1); - EntityKey sortProperty = new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME); - entityDataPageLink.setSortOrder(new EntityDataSortOrder(sortProperty, EntityDataSortOrder.Direction.DESC)); - EntityDataQuery query = new EntityDataQuery(entityTypeFilter, entityDataPageLink, List.of(sortProperty), Collections.emptyList(), Collections.emptyList()); - - DaoUtil.processInBatches(pageLink -> { - entityDataPageLink.setPage(pageLink.getPage()); - entityDataPageLink.setPageSize(pageLink.getPageSize()); - return entityService.findEntityDataByQuery(user.getTenantId(), new CustomerId(EntityId.NULL_UUID), query); - }, 100, data -> { - EntityId entityId = data.getEntityId(); - try { - saveEntityData(user, repository, entityId, config); - } catch (Exception e) { - throw new RuntimeException(e); + if (config.isAllEntities()) { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + try { + saveEntityData(user, commit, entity.getId(), config); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } else { + for (UUID entityId : config.getEntityIds()) { + try { + saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config); + } catch (Exception e) { + throw new RuntimeException(e); + } } - }); + } + }); break; } } - repository.add("."); - - VersionCreationResult result = new VersionCreationResult(); - GitRepository.Status status = repository.status(); - result.setAdded(status.getAdded().size()); - result.setModified(status.getModified().size()); - result.setRemoved(status.getRemoved().size()); - - GitRepository.Commit commit = repository.commit(request.getVersionName()); - repository.push(); - - result.setVersion(toVersion(commit)); - return result; + return gitService.push(commit); } - private void saveEntityData(SecurityUser user, GitRepository repository, EntityId entityId, VersionCreateConfig config) throws Exception { + private void saveEntityData(SecurityUser user, PendingCommit commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) .build()); - String entityDataJson = jsonWriter.writeValueAsString(entityData); - FileUtils.write(Path.of(repository.getDirectory(), getRelativePath(entityData.getEntityType(), - entityData.getEntity().getId().toString())).toFile(), entityDataJson, StandardCharsets.UTF_8); + gitService.addToCommit(commit, entityData); } @Override public List listEntityVersions(TenantId tenantId, String branch, EntityId externalId) throws Exception { - return listVersions(tenantId, branch, getRelativePath(externalId.getEntityType(), externalId.getId().toString())); + return gitService.listVersions(tenantId, branch, externalId); } @Override public List listEntityTypeVersions(TenantId tenantId, String branch, EntityType entityType) throws Exception { - return listVersions(tenantId, branch, getRelativePath(entityType, null)); + return gitService.listVersions(tenantId, branch, entityType); } @Override public List listVersions(TenantId tenantId, String branch) throws Exception { - return listVersions(tenantId, branch, null); + return gitService.listVersions(tenantId, branch); } - private List listVersions(TenantId tenantId, String branch, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() - .map(this::toVersion) - .collect(Collectors.toList()); - } - - @Override - public List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, getRelativePath(entityType, null)); + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception { + return gitService.listEntitiesAtVersion(tenantId, branch, versionId, entityType); } @Override public List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listEntitiesAtVersion(tenantId, branch, versionId, null); + return gitService.listEntitiesAtVersion(tenantId, branch, versionId); } - private List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { - GitRepository repository = checkRepository(tenantId); - checkVersion(tenantId, branch, versionId); - return repository.listFilesAtCommit(versionId, path).stream() - .map(filePath -> { - EntityId entityId = fromRelativePath(filePath); - VersionedEntityInfo info = new VersionedEntityInfo(); - info.setExternalId(entityId); - return info; - }) - .collect(Collectors.toList()); - } - - @Override public List loadEntitiesVersion(SecurityUser user, VersionLoadRequest request) throws Exception { - GitRepository repository = checkRepository(user.getTenantId()); - - EntityVersion version = checkVersion(user.getTenantId(), request.getBranch(), request.getVersionId()); - switch (request.getType()) { case SINGLE_ENTITY: { SingleEntityVersionLoadRequest versionLoadRequest = (SingleEntityVersionLoadRequest) request; + VersionLoadConfig config = versionLoadRequest.getConfig(); EntityImportResult importResult = transactionTemplate.execute(status -> { try { - EntityImportResult result = loadEntity(user, repository, versionLoadRequest.getExternalEntityId(), version.getId(), versionLoadRequest.getConfig()); - result.getSaveReferencesCallback().run(); - return result; + EntityExportData entityData = gitService.getEntity(user.getTenantId(), request.getVersionId(), versionLoadRequest.getExternalEntityId()); + return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), true, true); } catch (Exception e) { throw new RuntimeException(e); } }); - try { - importResult.getSendEventsCallback().run(); - } catch (Exception e) { - log.error("Failed to send events for entity", e); - } - return List.of(VersionLoadResult.builder() + .entityType(importResult.getEntityType()) .created(importResult.getOldEntity() == null ? 1 : 0) .updated(importResult.getOldEntity() != null ? 1 : 0) .deleted(0) @@ -305,56 +187,68 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityTypeVersionLoadRequest versionLoadRequest = (EntityTypeVersionLoadRequest) request; return transactionTemplate.execute(status -> { Map results = new HashMap<>(); + Map> importedEntities = new HashMap<>(); List saveReferencesCallbacks = new ArrayList<>(); List sendEventsCallbacks = new ArrayList<>(); - // order entity types.. - // or what - versionLoadRequest.getEntityTypes().forEach((entityType, config) -> { - AtomicInteger created = new AtomicInteger(); - AtomicInteger updated = new AtomicInteger(); - AtomicInteger deleted = new AtomicInteger(); - - Set remoteEntities; - try { - remoteEntities = listEntitiesAtVersion(user.getTenantId(), request.getBranch(), request.getVersionId(), getRelativePath(entityType, null)).stream() - .map(VersionedEntityInfo::getExternalId) - .collect(Collectors.toSet()); - for (EntityId externalEntityId : remoteEntities) { - EntityImportResult importResult = loadEntity(user, repository, externalEntityId, version.getId(), config); - - if (importResult.getOldEntity() == null) created.incrementAndGet(); - else updated.incrementAndGet(); - saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); - sendEventsCallbacks.add(importResult.getSendEventsCallback()); - } - } catch (Exception e) { - throw new RuntimeException(e); - } + versionLoadRequest.getEntityTypes().keySet().stream() + .sorted(exportImportService.getEntityTypeComparatorForImport()) + .forEach(entityType -> { + EntityTypeVersionLoadConfig config = versionLoadRequest.getEntityTypes().get(entityType); + AtomicInteger created = new AtomicInteger(); + AtomicInteger updated = new AtomicInteger(); - if (config.isRemoveOtherEntities()) { - DaoUtil.processInBatches(pageLink -> { - return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); - }, 100, entity -> { - if (entity.getExternalId() == null || !remoteEntities.contains(entity.getExternalId())) { - try { - exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); - } catch (ThingsboardException e) { - throw new RuntimeException(e); - } - // need to delete entity types in a specific order? - exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); - deleted.getAndIncrement(); + try { + int limit = 100; + int offset = 0; + List> entityDataList; + do { + entityDataList = gitService.getEntities(user.getTenantId(), request.getBranch(), request.getVersionId(), entityType, offset, limit); + for (EntityExportData entityData : entityDataList) { + EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() + .updateRelations(config.isLoadRelations()) + .findExistingByName(config.isFindExistingEntityByName()) + .build(), false, false); + + if (importResult.getOldEntity() == null) created.incrementAndGet(); + else updated.incrementAndGet(); + saveReferencesCallbacks.add(importResult.getSaveReferencesCallback()); + sendEventsCallbacks.add(importResult.getSendEventsCallback()); + } + offset += limit; + importedEntities.computeIfAbsent(entityType, t -> new HashSet<>()) + .addAll(entityDataList.stream().map(entityData -> entityData.getEntity().getId()).collect(Collectors.toSet())); + } while (entityDataList.size() == limit); + } catch (Exception e) { + throw new RuntimeException(e); } + results.put(entityType, VersionLoadResult.builder() + .entityType(entityType) + .created(created.get()) + .updated(updated.get()) + .build()); }); - } - results.put(entityType, VersionLoadResult.builder() - .created(created.get()) - .updated(updated.get()) - .deleted(deleted.get()) - .build()); - }); + versionLoadRequest.getEntityTypes().keySet().stream() + .filter(entityType -> versionLoadRequest.getEntityTypes().get(entityType).isRemoveOtherEntities()) + .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) + .forEach(entityType -> { + DaoUtil.processInBatches(pageLink -> { + return exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink); + }, 100, entity -> { + if (entity.getExternalId() == null || !importedEntities.get(entityType).contains(entity.getExternalId())) { + try { + exportableEntitiesService.checkPermission(user, entity, entityType, Operation.DELETE); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + + VersionLoadResult result = results.get(entityType); + result.setDeleted(result.getDeleted() + 1); + } + }); + }); for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { try { @@ -378,21 +272,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } } - private EntityImportResult loadEntity(SecurityUser user, GitRepository repository, EntityId externalId, String versionId, VersionLoadConfig config) throws Exception { - String entityDataJson = repository.getFileContentAtCommit(getRelativePath(externalId.getEntityType(), externalId.getId().toString()), versionId); - EntityExportData entityData = JacksonUtil.fromString(entityDataJson, EntityExportData.class); - - return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() - .updateRelations(config.isLoadRelations()) - .findExistingByName(config.isFindExistingEntityByName()) - .build(), false, false); - } - @Override public List listBranches(TenantId tenantId) throws Exception { - GitRepository repository = checkRepository(tenantId); - return repository.listBranches(); + return gitService.listBranches(tenantId); } @Override @@ -424,8 +307,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new RuntimeException("Failed to load version control settings!", e); } try { - clearRepository(tenantId); - initRepository(tenantId, savedVersionControlSettings); + gitService.initRepository(tenantId, savedVersionControlSettings); } catch (Exception e) { throw new RuntimeException("Failed to init repository!", e); } @@ -436,10 +318,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont public void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException { EntitiesVersionControlSettings storedSettings = getVersionControlSettings(tenantId); settings = this.restoreCredentials(settings, storedSettings); - Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); try { - GitRepository.test(settings, repositoryDirectory.toFile()); - } catch (GitAPIException e) { + gitService.testRepository(tenantId, settings); + } catch (Exception e) { throw new ThingsboardException(String.format("Unable to access repository: %s", e.getMessage()), ThingsboardErrorCode.GENERAL); } @@ -462,53 +343,4 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } return settings; } - - private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { - return listVersions(tenantId, branch, null).stream() - .filter(version -> version.getId().equals(versionId)) - .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); - } - - private GitRepository checkRepository(TenantId tenantId) { - return Optional.ofNullable(repositories.get(tenantId)) - .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); - } - - private void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { - Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); - - FileUtils.forceDelete(repositoryDirectory.toFile()); - Files.createDirectories(repositoryDirectory); - - GitRepository repository = GitRepository.clone(settings, repositoryDirectory.toFile()); - - repositories.put(tenantId, repository); - } - - private void clearRepository(TenantId tenantId) throws IOException { - GitRepository repository = repositories.get(tenantId); - if (repository != null) { - FileUtils.deleteDirectory(new File(repository.getDirectory())); - repositories.remove(tenantId); - } - } - - private EntityVersion toVersion(GitRepository.Commit commit) { - return new EntityVersion(commit.getId(), commit.getMessage()); - } - - private String getRelativePath(EntityType entityType, String entityId) { - String path = entityType.name().toLowerCase(); - if (entityId != null) { - path += "/" + entityId + ".json"; - } - return path; - } - - private EntityId fromRelativePath(String path) { - EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); - String entityId = StringUtils.substringBetween(path, "/", ".json"); - return EntityIdFactory.getByTypeAndUuid(entityType, entityId); - } - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java index 2854ec1d05..8c18aeb32d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java @@ -19,14 +19,14 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.sync.vc.data.EntityVersion; -import org.thingsboard.server.service.sync.vc.data.VersionCreationResult; -import org.thingsboard.server.service.sync.vc.data.VersionLoadResult; -import org.thingsboard.server.service.sync.vc.data.VersionedEntityInfo; -import org.thingsboard.server.service.sync.vc.data.request.create.VersionCreateRequest; -import org.thingsboard.server.service.sync.vc.data.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionLoadResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; import java.util.List; @@ -42,7 +42,7 @@ public interface EntitiesVersionControlService { List listVersions(TenantId tenantId, String branch) throws Exception; - List listEntitiesAtVersion(TenantId tenantId, EntityType entityType, String branch, String versionId) throws Exception; + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) throws Exception; List listAllEntitiesAtVersion(TenantId tenantId, String branch, String versionId) throws Exception; @@ -58,6 +58,4 @@ public interface EntitiesVersionControlService { void checkVersionControlAccess(TenantId tenantId, EntitiesVersionControlSettings settings) throws ThingsboardException; - - } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java new file mode 100644 index 0000000000..aa51fc3ea1 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LocalGitVersionControlService.java @@ -0,0 +1,266 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.dao.DaoUtil; +import org.thingsboard.server.dao.attributes.AttributesService; +import org.thingsboard.server.dao.tenant.TenantDao; +import org.thingsboard.server.queue.util.AfterStartUp; + +import java.io.IOException; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Slf4j +@RequiredArgsConstructor +@Service +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +public class LocalGitVersionControlService implements GitVersionControlService { + + private final ObjectWriter jsonWriter = new ObjectMapper().writer(SerializationFeature.INDENT_OUTPUT); + private final GitRepositoryService gitRepositoryService; + private final TenantDao tenantDao; + private final EntitiesVersionControlService entitiesVersionControlService; + private final ConcurrentMap tenantRepoLocks = new ConcurrentHashMap<>(); + private final Map pendingCommitMap = new HashMap<>(); + + @AfterStartUp + public void init() { + DaoUtil.processInBatches(tenantDao::findTenantsIds, 100, tenantId -> { + EntitiesVersionControlSettings settings = entitiesVersionControlService.getVersionControlSettings(tenantId); + if (settings != null) { + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + log.warn("Failed to init repository for tenant {}", tenantId, e); + } + } + }); + } + + @Override + public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.testRepository(tenantId, settings); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + gitRepositoryService.initRepository(tenantId, settings); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } finally { + lock.unlock(); + } + } + + @Override + public PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request) { + var lock = getRepoLock(tenantId); + lock.lock(); + try { + var pendingCommit = new PendingCommit(tenantId, request); + PendingCommit old = pendingCommitMap.put(tenantId, pendingCommit); + if (old != null) { + gitRepositoryService.abort(old); + } + gitRepositoryService.prepareCommit(pendingCommit); + return pendingCommit; + } finally { + lock.unlock(); + } + } + + @Override + public void deleteAll(PendingCommit commit, EntityType entityType) { + doInsideLock(commit, c -> { + try { + gitRepositoryService.deleteFolderContent(commit, getRelativePath(entityType, null)); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public void addToCommit(PendingCommit commit, EntityExportData> entityData) { + doInsideLock(commit, c -> { + String entityDataJson; + try { + entityDataJson = jsonWriter.writeValueAsString(entityData); + gitRepositoryService.add(c, getRelativePath(entityData.getEntityType(), + entityData.getEntity().getId().toString()), entityDataJson); + } catch (IOException e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + }); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + return executeInsideLock(commit, gitRepositoryService::push); + } + + @Override + public List listVersions(TenantId tenantId, String branch) { + return listVersions(tenantId, branch, (String) null); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityType entityType) { + return listVersions(tenantId, branch, getRelativePath(entityType, null)); + } + + @Override + public List listVersions(TenantId tenantId, String branch, EntityId entityId) { + return listVersions(tenantId, branch, getRelativePath(entityId.getEntityType(), entityId.getId().toString())); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { + try { + return gitRepositoryService.listEntitiesAtVersion(tenantId, branch, versionId, entityType != null ? getRelativePath(entityType, null) : null); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { + return listEntitiesAtVersion(tenantId, branch, versionId, null); + } + + @Override + public List listBranches(TenantId tenantId) { + return gitRepositoryService.listBranches(tenantId); + } + + @Override + public List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit) { + return listEntitiesAtVersion(tenantId, branch, versionId, entityType).stream() + .skip(offset).limit(limit) + .map(entityInfo -> getEntity(tenantId, versionId, entityInfo.getExternalId())) + .collect(Collectors.toList()); + } + + @Override + public EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId) { + try { + String entityDataJson = gitRepositoryService.getFileContentAtCommit(tenantId, + getRelativePath(entityId.getEntityType(), entityId.getId().toString()), versionId); + return JacksonUtil.fromString(entityDataJson, EntityExportData.class); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private List listVersions(TenantId tenantId, String branch, String path) { + try { + return gitRepositoryService.listVersions(tenantId, branch, path); + } catch (Exception e) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(e); + } + } + + private void doInsideLock(PendingCommit commit, Consumer r) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + r.accept(commit); + } finally { + lock.unlock(); + } + } + + private T executeInsideLock(PendingCommit commit, Function c) { + var lock = getRepoLock(commit.getTenantId()); + lock.lock(); + try { + checkCommit(commit); + return c.apply(commit); + } finally { + lock.unlock(); + } + } + + private void checkCommit(PendingCommit commit) { + PendingCommit existing = pendingCommitMap.get(commit.getTenantId()); + if (existing == null || !existing.getTxId().equals(commit.getTxId())) { + throw new ConcurrentModificationException(); + } + } + + private String getRelativePath(EntityType entityType, String entityId) { + String path = entityType.name().toLowerCase(); + if (entityId != null) { + path += "/" + entityId + ".json"; + } + return path; + } + + private Lock getRepoLock(TenantId tenantId) { + return tenantRepoLocks.computeIfAbsent(tenantId, t -> new ReentrantLock()); + } + +} diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 03dbb49763..654929f500 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1115,6 +1115,11 @@ metrics: # Metrics percentiles returned by actuator for timer metrics. List of double values (divided by ,). percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" +vc: + git: + service: "${JS_VC_GIT_SERVICE:local}" # local/remote + repos-poll-interval: "${TB_VC_GIT_REPOS_POLL_INTERVAL_SEC:60}" + management: endpoints: web: diff --git a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java similarity index 97% rename from application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java index cfc38ff3c3..9d326246d4 100644 --- a/application/src/main/java/org/thingsboard/server/utils/JsonTbEntity.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/JsonTbEntity.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import com.fasterxml.jackson.annotation.JacksonAnnotationsInside; import com.fasterxml.jackson.annotation.JsonSubTypes; diff --git a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java index 858ccdf422..1cb2ac8c74 100644 --- a/application/src/main/java/org/thingsboard/server/utils/ThrowingRunnable.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ThrowingRunnable.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.common.data.sync; import org.thingsboard.server.common.data.exception.ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java index 4be338d7cc..faf9280b93 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/DeviceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/DeviceExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 81e3b344d0..b361a302fc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; -import org.thingsboard.server.utils.JsonTbEntity; +import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 0fb318b5fb..051745c07f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java index 242856bfcb..ea69425a2c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportResult.java @@ -13,15 +13,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; -import org.thingsboard.server.utils.JsonTbEntity; -import org.thingsboard.server.utils.ThrowingRunnable; +import org.thingsboard.server.common.data.sync.JsonTbEntity; +import org.thingsboard.server.common.data.sync.ThrowingRunnable; @Data public class EntityImportResult> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 075a6896ef..3e18bc9f5f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/importing/data/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.importing.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java index e1d870473f..b176782f62 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/exportimport/exporting/data/RuleChainExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/RuleChainExportData.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.exportimport.exporting.data; +package org.thingsboard.server.common.data.sync.ie; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java similarity index 94% rename from common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java index ce4f8e71d2..4fdff1f413 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/vc/EntitiesVersionControlSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntitiesVersionControlSettings.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.vc; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; - @Data public class EntitiesVersionControlSettings { private String repositoryUri; @@ -27,4 +26,4 @@ public class EntitiesVersionControlSettings { private String privateKey; private String privateKeyPassword; private String defaultBranch; -} +} \ No newline at end of file diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java index 3547f89e3c..8779d64ed3 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntityVersion.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityVersion.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java similarity index 93% rename from common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java index 9d2dfa048e..770a1e582c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/vc/VersionControlAuthMethod.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionControlAuthMethod.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.common.data.vc; +package org.thingsboard.server.common.data.sync.vc; public enum VersionControlAuthMethod { USERNAME_PASSWORD, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java index 9194c67272..8cb09b1b30 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionCreationResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionCreationResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java index cf8b588008..0039546fb0 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionLoadResult.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.AllArgsConstructor; import lombok.Builder; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index 0a65297892..163fe4c6d2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data; +package org.thingsboard.server.common.data.sync.vc; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java index 3c34680f95..f8d7279f40 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/ComplexVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/ComplexVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java index 1c27d9d2f1..92cab96354 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityTypeVersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/EntityTypeVersionCreateConfig.java @@ -13,13 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; import java.util.List; +import java.util.UUID; @Data @EqualsAndHashCode(callSuper = true) @@ -27,7 +27,7 @@ public class EntityTypeVersionCreateConfig extends VersionCreateConfig { //optional private SyncStrategy syncStrategy; - private List entityIds; + private List entityIds; private boolean allEntities; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java similarity index 93% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java index 4d03404199..507afd116e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SingleEntityVersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SingleEntityVersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java similarity index 90% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java index 43ec873f22..baf24efeb8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/SyncStrategy.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/SyncStrategy.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum SyncStrategy { MERGE, diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index d4f8354df4..a1aa6cd6d8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java similarity index 83% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java index 10f093d26d..9a23838921 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; @@ -23,8 +23,7 @@ import lombok.Data; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({ @Type(name = "SINGLE_ENTITY", value = SingleEntityVersionCreateRequest.class), - @Type(name = "ENTITY_LIST", value = EntityListVersionCreateRequest.class), - @Type(name = "ENTITY_TYPE", value = ComplexVersionCreateRequest.class) + @Type(name = "COMPLEX", value = ComplexVersionCreateRequest.class) }) @Data public abstract class VersionCreateRequest { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java index fea8b6fec7..d57363cf7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/VersionCreateRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateRequestType.java @@ -13,10 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.common.data.sync.vc.request.create; public enum VersionCreateRequestType { SINGLE_ENTITY, - ENTITY_LIST, COMPLEX } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java index 27597f5e87..837986f82d 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java index 0ce902c83e..f2cb83f0d4 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/EntityTypeVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/EntityTypeVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java index eb4bb1bdc1..cf31317b4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/SingleEntityVersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/SingleEntityVersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java similarity index 92% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 6ead76f105..31b34dbcb7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import lombok.Data; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java index 5a24bc78e5..d9d1329c8b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequest.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequest.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java similarity index 91% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java rename to common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java index 6a59cba676..190e6871d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/load/VersionLoadRequestType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadRequestType.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.load; +package org.thingsboard.server.common.data.sync.vc.request.load; public enum VersionLoadRequestType { SINGLE_ENTITY, diff --git a/common/pom.xml b/common/pom.xml index 236fb50a19..62bb46e81c 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -46,6 +46,7 @@ cache coap-server edge-api + version-control diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml new file mode 100644 index 0000000000..b01b466cb5 --- /dev/null +++ b/common/version-control/pom.xml @@ -0,0 +1,119 @@ + + + 4.0.0 + + org.thingsboard + 3.4.0-SNAPSHOT + common + + org.thingsboard.common + version-control + jar + + Thingsboard Server Version Control API + https://thingsboard.io + + + UTF-8 + ${basedir}/../.. + + + + + org.springframework + spring-core + + + org.springframework + spring-context-support + + + org.springframework + spring-context + + + org.springframework.boot + spring-boot-starter-web + provided + + + javax.annotation + javax.annotation-api + + + com.google.guava + guava + provided + + + com.fasterxml.jackson.core + jackson-databind + + + org.slf4j + slf4j-api + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-core + + + ch.qos.logback + logback-classic + + + org.eclipse.jgit + org.eclipse.jgit + + + org.eclipse.jgit + org.eclipse.jgit.ssh.apache + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.awaitility + awaitility + test + + + org.thingsboard.common + data + + + + + + + + + diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java new file mode 100644 index 0000000000..56ca38ffe4 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java @@ -0,0 +1,245 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.api.errors.JGitInternalException; +import org.eclipse.jgit.api.errors.RefAlreadyExistsException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Slf4j +@ConditionalOnProperty(prefix = "vc", value = "git.service", havingValue = "local", matchIfMissing = true) +@Service +public class DefaultGitRepositoryService implements GitRepositoryService { + + @Value("${vc.git.repos-poll-interval:${java.io.tmpdir}/repositories}") + private String repositoriesFolder; + + @Value("${vc.git.repos-poll-interval:60}") + private long reposPollInterval; + + private ScheduledExecutorService scheduler; + private final Map repositories = new ConcurrentHashMap<>(); + + @PostConstruct + public void init() { + scheduler = Executors.newSingleThreadScheduledExecutor(); + scheduler.scheduleWithFixedDelay(() -> { + repositories.forEach((tenantId, repository) -> { + try { + repository.fetch(); + log.info("Fetching remote repository for tenant {}", tenantId); + } catch (Exception e) { + log.warn("Failed to fetch repository for tenant {}", tenantId, e); + } + }); + }, reposPollInterval, reposPollInterval, TimeUnit.SECONDS); + } + + @PreDestroy + public void stop() { + if (scheduler != null) { + scheduler.shutdownNow(); + } + } + + @Override + public void prepareCommit(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + String branch = commit.getRequest().getBranch(); + try { + repository.fetch(); + if (repository.listBranches().contains(branch)) { + repository.checkout("origin/" + branch, false); + try { + repository.checkout(branch, true); + } catch (RefAlreadyExistsException e) { + repository.checkout(branch, false); + } + repository.merge(branch); + } else { // TODO [viacheslav]: rollback orphan branch on failure + try { + repository.createAndCheckoutOrphanBranch(branch); // FIXME [viacheslav]: Checkout returned unexpected result NO_CHANGE for master branch + } catch (JGitInternalException e) { + if (!e.getMessage().contains("NO_CHANGE")) { + throw e; + } + } + } + } catch (IOException | GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.deleteDirectory(Path.of(repository.getDirectory(), relativePath).toFile()); + } + + @Override + public void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException { + GitRepository repository = checkRepository(commit.getTenantId()); + FileUtils.write(Path.of(repository.getDirectory(), relativePath).toFile(), entityDataJson, StandardCharsets.UTF_8); + } + + @Override + public VersionCreationResult push(PendingCommit commit) { + GitRepository repository = checkRepository(commit.getTenantId()); + try { + repository.add("."); + + VersionCreationResult result = new VersionCreationResult(); + GitRepository.Status status = repository.status(); + result.setAdded(status.getAdded().size()); + result.setModified(status.getModified().size()); + result.setRemoved(status.getRemoved().size()); + + GitRepository.Commit gitCommit = repository.commit(commit.getRequest().getVersionName()); + repository.push(); + + result.setVersion(toVersion(gitCommit)); + return result; + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + @Override + public void abort(PendingCommit commit) { + //TODO: implement; + } + + @Override + public String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException { + GitRepository repository = checkRepository(tenantId); + return repository.getFileContentAtCommit(relativePath, versionId); + } + + @Override + public List listBranches(TenantId tenantId) { + GitRepository repository = checkRepository(tenantId); + try { + return repository.listBranches(); + } catch (GitAPIException gitAPIException) { + //TODO: analyze and return meaningful exceptions that we can show to the client; + throw new RuntimeException(gitAPIException); + } + } + + private EntityVersion checkVersion(TenantId tenantId, String branch, String versionId) throws Exception { + return listVersions(tenantId, branch, null).stream() + .filter(version -> version.getId().equals(versionId)) + .findFirst().orElseThrow(() -> new IllegalArgumentException("Version not found")); + } + + private GitRepository checkRepository(TenantId tenantId) { + return Optional.ofNullable(repositories.get(tenantId)) + .orElseThrow(() -> new IllegalStateException("Repository is not initialized")); + } + + @Override + public List listVersions(TenantId tenantId, String branch, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + return repository.listCommits(branch, path, Integer.MAX_VALUE).stream() + .map(this::toVersion) + .collect(Collectors.toList()); + } + + @Override + public List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception { + GitRepository repository = checkRepository(tenantId); + checkVersion(tenantId, branch, versionId); + return repository.listFilesAtCommit(versionId, path).stream() + .map(filePath -> { + EntityId entityId = fromRelativePath(filePath); + VersionedEntityInfo info = new VersionedEntityInfo(); + info.setExternalId(entityId); + return info; + }) + .collect(Collectors.toList()); + } + + @Override + public void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository.test(settings, repositoryDirectory.toFile()); + } + + @Override + public void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception { + Path repositoryDirectory = Path.of(repositoriesFolder, tenantId.getId().toString()); + GitRepository repository; + if (Files.exists(repositoryDirectory)) { + FileUtils.forceDelete(repositoryDirectory.toFile()); + } + + Files.createDirectories(repositoryDirectory); + repository = GitRepository.clone(settings, repositoryDirectory.toFile()); + repositories.put(tenantId, repository); + } + + private void clearRepository(TenantId tenantId) throws IOException { + GitRepository repository = repositories.get(tenantId); + if (repository != null) { + FileUtils.deleteDirectory(new File(repository.getDirectory())); + repositories.remove(tenantId); + } + } + + + private EntityVersion toVersion(GitRepository.Commit commit) { + return new EntityVersion(commit.getId(), commit.getMessage()); + } + + private EntityId fromRelativePath(String path) { + EntityType entityType = EntityType.valueOf(StringUtils.substringBefore(path, "/").toUpperCase()); + String entityId = StringUtils.substringBetween(path, "/", ".json"); + return EntityIdFactory.getByTypeAndUuid(entityType, entityId); + } +} diff --git a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java similarity index 94% rename from application/src/main/java/org/thingsboard/server/utils/GitRepository.java rename to common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index e207c30fa2..fa4891d35a 100644 --- a/application/src/main/java/org/thingsboard/server/utils/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.utils; +package org.thingsboard.server.service.sync.vc; import com.google.common.collect.Streams; import lombok.Data; @@ -37,8 +37,8 @@ import org.eclipse.jgit.transport.sshd.SshdSessionFactoryBuilder; import org.eclipse.jgit.treewalk.TreeWalk; import org.eclipse.jgit.treewalk.filter.PathFilter; import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.vc.EntitiesVersionControlSettings; -import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.VersionControlAuthMethod; import java.io.ByteArrayInputStream; import java.io.File; @@ -116,9 +116,15 @@ public class GitRepository { .setRemoveDeletedRefs(true)); } - public void checkout(String branch) throws GitAPIException { + public void checkout(String branch, boolean createBranch) throws GitAPIException { execute(git.checkout() - .setName("origin/" + branch)); + .setCreateBranch(createBranch) + .setName(branch)); + } + + public void reset() throws GitAPIException { + execute(git.reset() + .setMode(ResetCommand.ResetType.HARD)); } public void merge(String branch) throws IOException, GitAPIException { @@ -201,14 +207,15 @@ public class GitRepository { public void createAndCheckoutOrphanBranch(String name) throws GitAPIException { execute(git.checkout() .setOrphan(true) + .setForced(true) .setName(name)); - Set uncommittedChanges = git.status().call().getUncommittedChanges(); - if (!uncommittedChanges.isEmpty()) { - RmCommand rm = git.rm(); - uncommittedChanges.forEach(rm::addFilepattern); - execute(rm); - } - execute(git.clean()); +// Set uncommittedChanges = git.status().call().getUncommittedChanges(); +// if (!uncommittedChanges.isEmpty()) { +// RmCommand rm = git.rm(); +// uncommittedChanges.forEach(rm::addFilepattern); +// execute(rm); +// } +// execute(git.clean()); } public void add(String filesPattern) throws GitAPIException { // FIXME [viacheslav] diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java new file mode 100644 index 0000000000..4b1843e3c7 --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java @@ -0,0 +1,50 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; + +import java.io.IOException; +import java.util.List; + +public interface GitRepositoryService { + + void prepareCommit(PendingCommit pendingCommit); + + List listVersions(TenantId tenantId, String branch, String path) throws Exception; + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, String path) throws Exception; + + void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings) throws Exception; + + void add(PendingCommit commit, String relativePath, String entityDataJson) throws IOException; + + void deleteFolderContent(PendingCommit commit, String relativePath) throws IOException; + + VersionCreationResult push(PendingCommit commit); + + void abort(PendingCommit commit); + + List listBranches(TenantId tenantId); + + String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException; +} diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java new file mode 100644 index 0000000000..8345375e4f --- /dev/null +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlService.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.vc.EntitiesVersionControlSettings; +import org.thingsboard.server.common.data.sync.vc.EntityVersion; +import org.thingsboard.server.common.data.sync.vc.VersionCreationResult; +import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; + +import java.util.List; + +public interface GitVersionControlService { + + void testRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + void initRepository(TenantId tenantId, EntitiesVersionControlSettings settings); + + PendingCommit prepareCommit(TenantId tenantId, VersionCreateRequest request); + + void addToCommit(PendingCommit commit, EntityExportData> entityData); + + void deleteAll(PendingCommit pendingCommit, EntityType entityType); + + VersionCreationResult push(PendingCommit commit); + + List listVersions(TenantId tenantId, String branch); + + List listVersions(TenantId tenantId, String branch, EntityType entityType); + + List listVersions(TenantId tenantId, String branch, EntityId entityId); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType); + + List listEntitiesAtVersion(TenantId tenantId, String branch, String versionId); + + List listBranches(TenantId tenantId); + + EntityExportData getEntity(TenantId tenantId, String versionId, EntityId entityId); + + List> getEntities(TenantId tenantId, String branch, String versionId, EntityType entityType, int offset, int limit); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java similarity index 54% rename from application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java rename to common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index adeac70e89..a30f7514a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/request/create/EntityListVersionCreateRequest.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -13,24 +13,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.sync.vc.data.request.create; +package org.thingsboard.server.service.sync.vc; import lombok.Data; -import lombok.EqualsAndHashCode; -import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; -import java.util.List; +import java.util.UUID; @Data -@EqualsAndHashCode(callSuper = true) -public class EntityListVersionCreateRequest extends VersionCreateRequest { +public class PendingCommit { - private List entitiesIds; - private EntityTypeVersionCreateConfig config; + private final UUID txId; + private final TenantId tenantId; + private final VersionCreateRequest request; - @Override - public VersionCreateRequestType getType() { - return VersionCreateRequestType.ENTITY_LIST; + public PendingCommit(TenantId tenantId, VersionCreateRequest request) { + this.txId = UUID.randomUUID(); + this.tenantId = tenantId; + this.request = request; } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java index 0462cc38de..0bb5564fa6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/settings/AdminSettingsServiceImpl.java @@ -22,7 +22,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.AdminSettingsId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.vc.VersionControlAuthMethod; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.Validator; @@ -59,22 +58,6 @@ public class AdminSettingsServiceImpl implements AdminSettingsService { if (mailSettings != null) { ((ObjectNode) adminSettings.getJsonValue()).put("password", mailSettings.getJsonValue().get("password").asText()); } - } else if (adminSettings.getKey().equals("entitiesVersionControl")) { - VersionControlAuthMethod authMethod = VersionControlAuthMethod.valueOf(adminSettings.getJsonValue().get("authMethod").asText()); - if (VersionControlAuthMethod.USERNAME_PASSWORD.equals(authMethod) && !adminSettings.getJsonValue().has("password")) { - AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); - if (vcSettings != null) { - ((ObjectNode) adminSettings.getJsonValue()).put("password", vcSettings.getJsonValue().get("password").asText()); - } - } else if (VersionControlAuthMethod.PRIVATE_KEY.equals(authMethod) && !adminSettings.getJsonValue().has("privateKey")) { - AdminSettings vcSettings = findAdminSettingsByKey(tenantId, "entitiesVersionControl"); - if (vcSettings != null) { - ((ObjectNode) adminSettings.getJsonValue()).put("privateKey", vcSettings.getJsonValue().get("privateKey").asText()); - if (!adminSettings.getJsonValue().has("privateKeyPassword") && vcSettings.getJsonValue().has("privateKeyPassword")) { - ((ObjectNode) adminSettings.getJsonValue()).put("privateKeyPassword", vcSettings.getJsonValue().get("privateKeyPassword").asText()); - } - } - } } return adminSettingsDao.save(tenantId, adminSettings); } diff --git a/pom.xml b/pom.xml index 319f091769..15959886e0 100755 --- a/pom.xml +++ b/pom.xml @@ -866,6 +866,11 @@ util ${project.version} + + org.thingsboard.common + version-control + ${project.version} + org.thingsboard.common cache