From 8e3054c999d34e394589483e9985545ea562da74 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Wed, 15 Jun 2022 09:42:52 +0300 Subject: [PATCH] Entities Import Context --- .../DefaultEntitiesExportImportService.java | 7 ++-- .../sync/ie/EntitiesExportImportService.java | 3 +- .../ie/exporting/EntityExportService.java | 3 +- .../impl/BaseEntityExportService.java | 7 ++-- .../impl/DefaultEntityExportService.java | 23 ++++++----- .../DefaultEntitiesVersionControlService.java | 41 ++++++++++--------- .../vc/data/ComplexEntitiesExportCtx.java | 23 +++++++++++ .../sync/vc/data/EntitiesExportCtx.java | 37 +++++++++++++++++ .../sync/vc/data/EntitiesImportCtx.java | 9 ++-- .../sync/vc/data/SimpleEntitiesExportCtx.java | 17 ++++++++ 10 files changed, 127 insertions(+), 43 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java create mode 100644 application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 4794a659ab..094ce55cc7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -41,6 +41,7 @@ import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import java.util.ArrayList; @@ -71,15 +72,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @Override - public , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { - if (!rateLimitService.checkEntityExportLimit(user.getTenantId())) { + public , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException { + if (!rateLimitService.checkEntityExportLimit(ctx.getTenantId())) { throw new ThingsboardException("Rate limit for entities export is exceeded", ThingsboardErrorCode.TOO_MANY_REQUESTS); } EntityType entityType = entityId.getEntityType(); EntityExportService> exportService = getExportService(entityType); - return exportService.getExportData(user, entityId, exportSettings); + return exportService.getExportData(ctx, entityId, exportSettings); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java index 05e554f114..9031b1a51f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/EntitiesExportImportService.java @@ -24,13 +24,14 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import java.util.Comparator; public interface EntitiesExportImportService { - , I extends EntityId> EntityExportData exportEntity(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + , I extends EntityId> EntityExportData exportEntity(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; , I extends EntityId> EntityImportResult importEntity(EntitiesImportCtx ctx, EntityExportData exportData) throws ThingsboardException; diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java index 9b251a3dab..07f7d6d311 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/EntityExportService.java @@ -21,9 +21,10 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; public interface EntityExportService, D extends EntityExportData> { - D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; + D getExportData(EntitiesExportCtx ctx, I entityId, EntityExportSettings exportSettings) throws ThingsboardException; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 9e14531e48..1877672faa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -23,15 +23,16 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.Set; public abstract class BaseEntityExportService, D extends EntityExportData> extends DefaultEntityExportService { @Override - protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { - setRelatedEntities(user.getTenantId(), entity, (D) exportData, exportSettings); - super.setAdditionalExportData(user, entity, exportData, exportSettings); + protected void setAdditionalExportData(EntitiesExportCtx ctx, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException { + setRelatedEntities(ctx.getTenantId(), entity, (D) exportData, exportSettings); + super.setAdditionalExportData(ctx, entity, exportData, exportSettings); } protected void setRelatedEntities(TenantId tenantId, E mainEntity, D exportData, EntityExportSettings settings) {} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 1c1d70b0ae..ebcf89e261 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -36,6 +36,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import java.util.ArrayList; import java.util.Collections; @@ -59,44 +60,44 @@ public class DefaultEntityExportService relations = exportRelations(user, entity); + List relations = exportRelations(ctx, entity); exportData.setRelations(relations); } if (exportSettings.isExportAttributes()) { - Map> attributes = exportAttributes(user, entity); + Map> attributes = exportAttributes(ctx, entity); exportData.setAttributes(attributes); } } - private List exportRelations(SecurityUser user, E entity) throws ThingsboardException { + private List exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List relations = new ArrayList<>(); - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(inboundRelations); - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(outboundRelations); return relations; } - private Map> exportAttributes(SecurityUser user, E entity) throws ThingsboardException { + private Map> exportAttributes(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List scopes; if (entity.getId().getEntityType() == EntityType.DEVICE) { scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); @@ -106,7 +107,7 @@ public class DefaultEntityExportService> attributes = new LinkedHashMap<>(); scopes.forEach(scope -> { try { - attributes.put(scope, attributesService.findAll(user.getTenantId(), entity.getId(), scope).get().stream() + attributes.put(scope, attributesService.findAll(ctx.getTenantId(), entity.getId(), scope).get().stream() .map(attribute -> { AttributeExportData attributeExportData = new AttributeExportData(); attributeExportData.setKey(attribute.getKey()); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 6b299930d0..551edf7706 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -77,7 +77,10 @@ import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesServic import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; import org.thingsboard.server.service.sync.vc.autocommit.TbAutoCommitSettingsService; import org.thingsboard.server.service.sync.vc.data.CommitGitRequest; +import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx; +import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; +import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; import javax.annotation.PostConstruct; @@ -136,11 +139,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont List> gitFutures = new ArrayList<>(); switch (request.getType()) { case SINGLE_ENTITY: { - handleSingleEntityRequest(user, commit, gitFutures, (SingleEntityVersionCreateRequest) request); + handleSingleEntityRequest(new SimpleEntitiesExportCtx(user, commit, (SingleEntityVersionCreateRequest) request)); break; } case COMPLEX: { - handleComplexRequest(user, commit, gitFutures, (ComplexVersionCreateRequest) request); + handleComplexRequest(new ComplexEntitiesExportCtx(user, commit, (ComplexVersionCreateRequest) request)); break; } } @@ -148,21 +151,22 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }, executor); } - private void handleSingleEntityRequest(SecurityUser user, CommitGitRequest commit, List> gitFutures, SingleEntityVersionCreateRequest versionCreateRequest) throws Exception { - gitFutures.add(saveEntityData(user, commit, versionCreateRequest.getEntityId(), versionCreateRequest.getConfig())); + private void handleSingleEntityRequest(SimpleEntitiesExportCtx ctx) throws Exception { + ctx.add(saveEntityData(ctx, ctx.getRequest().getEntityId(), ctx.getSettings())); } - private void handleComplexRequest(SecurityUser user, CommitGitRequest commit, List> gitFutures, ComplexVersionCreateRequest versionCreateRequest) { - versionCreateRequest.getEntityTypes().forEach((entityType, config) -> { - if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), versionCreateRequest.getSyncStrategy()) == SyncStrategy.OVERWRITE) { - gitFutures.add(gitServiceQueue.deleteAll(commit, entityType)); + private void handleComplexRequest(ComplexEntitiesExportCtx ctx) { + ctx.getRequest().getEntityTypes().forEach((entityType, config) -> { + if (ObjectUtils.defaultIfNull(config.getSyncStrategy(), ctx.getRequest().getSyncStrategy()) == SyncStrategy.OVERWRITE) { + ctx.add(gitServiceQueue.deleteAll(ctx.getCommit(), entityType)); } + EntityExportSettings settings = ctx.getSettings(entityType); if (config.isAllEntities()) { - DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(user.getTenantId(), entityType, pageLink) + DaoUtil.processInBatches(pageLink -> exportableEntitiesService.findEntitiesByTenantId(ctx.getTenantId(), entityType, pageLink) , 100, entity -> { try { - gitFutures.add(saveEntityData(user, commit, entity.getId(), config)); + ctx.add(saveEntityData(ctx, entity.getId(), settings)); } catch (Exception e) { throw new RuntimeException(e); } @@ -170,7 +174,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } else { for (UUID entityId : config.getEntityIds()) { try { - gitFutures.add(saveEntityData(user, commit, EntityIdFactory.getByTypeAndUuid(entityType, entityId), config)); + ctx.add(saveEntityData(ctx, EntityIdFactory.getByTypeAndUuid(entityType, entityId), settings)); } catch (Exception e) { throw new RuntimeException(e); } @@ -179,13 +183,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }); } - private ListenableFuture saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { - EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() - .exportRelations(config.isSaveRelations()) - .exportAttributes(config.isSaveAttributes()) - .exportCredentials(config.isSaveCredentials()) - .build()); - return gitServiceQueue.addToCommit(commit, entityData); + private ListenableFuture saveEntityData(EntitiesExportCtx ctx, EntityId entityId, EntityExportSettings settings) throws Exception { + EntityExportData> entityData = exportImportService.exportEntity(ctx, entityId, settings); + return gitServiceQueue.addToCommit(ctx.getCommit(), entityData); } @Override @@ -410,7 +410,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> { - EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() + SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null); + EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId, EntityExportSettings.builder() .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java new file mode 100644 index 0000000000..673590d7f7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ComplexEntitiesExportCtx.java @@ -0,0 +1,23 @@ +package org.thingsboard.server.service.sync.vc.data; + +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.ComplexVersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.HashMap; +import java.util.Map; + +public class ComplexEntitiesExportCtx extends EntitiesExportCtx { + + private final Map settings = new HashMap<>(); + + public ComplexEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, ComplexVersionCreateRequest request) { + super(user, commit, request); + request.getEntityTypes().forEach((type, config) -> settings.put(type, buildExportSettings(config))); + } + + public EntityExportSettings getSettings(EntityType entityType) { + return settings.get(entityType); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java new file mode 100644 index 0000000000..0fa3261841 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesExportCtx.java @@ -0,0 +1,37 @@ +package org.thingsboard.server.service.sync.vc.data; + +import com.google.common.util.concurrent.ListenableFuture; +import lombok.Data; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateConfig; +import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class EntitiesExportCtx { + + protected final SecurityUser user; + protected final CommitGitRequest commit; + protected final R request; + private final List> futures = new ArrayList<>(); + + public void add(ListenableFuture future) { + futures.add(future); + } + + public TenantId getTenantId() { + return user.getTenantId(); + } + + protected static EntityExportSettings buildExportSettings(VersionCreateConfig config) { + return EntityExportSettings.builder() + .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) + .exportCredentials(config.isSaveCredentials()) + .build(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 94693009b4..497d2b9fab 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -40,11 +40,10 @@ public class EntitiesImportCtx { private final SecurityUser user; private final String versionId; - private EntityImportSettings settings; - Map results = new HashMap<>(); - Map> importedEntities = new HashMap<>(); - Map toReimport = new HashMap<>(); + private final Map results = new HashMap<>(); + private final Map> importedEntities = new HashMap<>(); + private final Map toReimport = new HashMap<>(); private final List referenceCallbacks = new ArrayList<>(); private final List eventCallbacks = new ArrayList<>(); @@ -52,6 +51,8 @@ public class EntitiesImportCtx { private final Set relations = new LinkedHashSet<>(); + private EntityImportSettings settings; + public EntitiesImportCtx(SecurityUser user, String versionId) { this(user, versionId, null); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java new file mode 100644 index 0000000000..600d70106b --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/SimpleEntitiesExportCtx.java @@ -0,0 +1,17 @@ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Getter; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.common.data.sync.vc.request.create.SingleEntityVersionCreateRequest; +import org.thingsboard.server.service.security.model.SecurityUser; + +public class SimpleEntitiesExportCtx extends EntitiesExportCtx { + + @Getter + private final EntityExportSettings settings; + + public SimpleEntitiesExportCtx(SecurityUser user, CommitGitRequest commit, SingleEntityVersionCreateRequest request) { + super(user, commit, request); + this.settings = request != null ? buildExportSettings(request.getConfig()) : null; + } +}