diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index df6b770d1f..8ef0ce8fc8 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -60,6 +60,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.stream.Collectors; @@ -280,40 +281,31 @@ public abstract class BaseEntityImportService getInternalIdByUuid(UUID externalUuid) { + public Optional getInternalIdByUuid(UUID externalUuid, boolean fetchAllUUIDs, Set hints) { if (externalUuid.equals(EntityId.NULL_UUID)) return Optional.empty(); for (EntityType entityType : EntityType.values()) { - EntityId externalId; - try { - externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid); - } catch (Exception e) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (!externalIdOpt.isPresent()) { continue; } - EntityId internalId = ctx.getInternalId(externalId); + EntityId internalId = ctx.getInternalId(externalIdOpt.get()); if (internalId != null) { return Optional.of(internalId); } } - for (EntityType entityType : EntityType.values()) { - EntityId externalId; - try { - externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid); - } catch (Exception e) { - continue; + if (fetchAllUUIDs) { + for (EntityType entityType : hints) { + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; } - - EntityId internalId = getInternalId(externalId, false); - if (internalId != null) { - return Optional.of(internalId); - } else if (ctx.isResetExternalIdsOfAnotherTenant()) { - try { - if (exportableEntitiesService.findEntityById(externalId) != null) { - return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, EntityId.NULL_UUID)); - } - } catch (Exception ignored) { + for (EntityType entityType : EntityType.values()) { + if (hints.contains(entityType)) { + continue; } + Optional internalId = lookupInDb(externalUuid, entityType); + if (internalId.isPresent()) return internalId; } } @@ -321,6 +313,28 @@ public abstract class BaseEntityImportService lookupInDb(UUID externalUuid, EntityType entityType) { + Optional externalIdOpt = buildEntityId(entityType, externalUuid); + if (externalIdOpt.isEmpty() || ctx.isNotFound(externalIdOpt.get())) { + return Optional.empty(); + } + EntityId internalId = getInternalId(externalIdOpt.get(), false); + if (internalId != null) { + return Optional.of(internalId); + } else { + ctx.registerNotFound(externalIdOpt.get()); + } + return Optional.empty(); + } + + private Optional buildEntityId(EntityType entityType, UUID externalUuid) { + try { + return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, externalUuid)); + } catch (Exception e) { + return Optional.empty(); + } + } + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index 24238c78e5..5c7e686c4a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -37,8 +37,10 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.utils.RegexUtils; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -49,6 +51,8 @@ import java.util.stream.Collectors; @RequiredArgsConstructor public class DashboardImportService extends BaseEntityImportService> { + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.DASHBOARD, EntityType.DEVICE, EntityType.ASSET)); + private final DashboardService dashboardService; @@ -78,7 +82,7 @@ public class DashboardImportService extends BaseEntityImportService { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) .map(entityId -> entityId.getId().toString()).orElse(uuid); })); ((ObjectNode) entityAlias).set(field, newFieldValue); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java index 4ffcb7a41c..d32f9589f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java @@ -35,7 +35,9 @@ import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.utils.RegexUtils; +import java.util.Arrays; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.Optional; import java.util.UUID; @@ -44,6 +46,8 @@ import java.util.UUID; @RequiredArgsConstructor public class RuleChainImportService extends BaseEntityImportService { + private static final LinkedHashSet HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET)); + private final RuleChainService ruleChainService; @Override @@ -70,7 +74,7 @@ public class RuleChainImportService extends BaseEntityImportService { - return idProvider.getInternalIdByUuid(UUID.fromString(uuid)) + return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS) .map(entityId -> entityId.getId().toString()) .orElse(uuid); }); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 69be12d580..61bce45a72 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -77,6 +77,7 @@ import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.sync.vc.data.EntityTypeExportCtx; +import org.thingsboard.server.service.sync.vc.data.ReimportTask; import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService; @@ -282,8 +283,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont for (EntityType entityType : entityTypes) { log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); sw.startNew("Entities " + entityType.name()); - EntityImportSettings settings = getEntityImportSettings(request, entityType); - ctx.setSettings(settings); + ctx.setSettings(getEntityImportSettings(request, entityType)); importEntities(ctx, entityType); } @@ -337,7 +337,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont throw new LoadEntityException(entityData, e); } if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) { - ctx.getToReimport().put(entityData.getEntity().getExternalId(), ctx.getSettings()); + ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(entityData, ctx.getSettings())); continue; } @@ -351,11 +351,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont @SuppressWarnings({"rawtypes", "unchecked"}) private void reimport(EntitiesImportCtx ctx) { - ctx.getToReimport().forEach((externalId, importSettings) -> { + ctx.setFetchAllUUIDs(true); + ctx.getToReimport().forEach((externalId, task) -> { try { - EntityExportData entityData = gitServiceQueue.getEntity(ctx.getTenantId(), ctx.getVersionId(), externalId).get(); - importSettings.setResetExternalIdsOfAnotherTenant(true); - ctx.setSettings(importSettings); + EntityExportData entityData = task.getData(); + var settings = task.getSettings(); + ctx.setSettings(settings); EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); ctx.registerResult(externalId.getEntityType(), importResult.getOldEntity() == null); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 497d2b9fab..74d09a5c3b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -29,6 +29,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -43,14 +44,15 @@ public class EntitiesImportCtx { private final Map results = new HashMap<>(); private final Map> importedEntities = new HashMap<>(); - private final Map toReimport = new HashMap<>(); + private final Map toReimport = new HashMap<>(); private final List referenceCallbacks = new ArrayList<>(); private final List eventCallbacks = new ArrayList<>(); - private final Map externalToInternalIdMap = new HashMap<>(); + private final Set notFoundIds = new HashSet<>(); private final Set relations = new LinkedHashSet<>(); + private boolean fetchAllUUIDs = false; private EntityImportSettings settings; public EntitiesImportCtx(SecurityUser user, String versionId) { @@ -83,10 +85,6 @@ public class EntitiesImportCtx { return getSettings().isSaveCredentials(); } - public boolean isResetExternalIdsOfAnotherTenant() { - return getSettings().isResetExternalIdsOfAnotherTenant(); - } - public EntityId getInternalId(EntityId externalId) { var result = externalToInternalIdMap.get(externalId); log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); @@ -128,5 +126,12 @@ public class EntitiesImportCtx { } } + public void registerNotFound(EntityId externalId) { + notFoundIds.add(externalId); + } + + public boolean isNotFound(EntityId externalId) { + return notFoundIds.contains(externalId); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java new file mode 100644 index 0000000000..97432adbb8 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.sync.vc.data; + +import lombok.Data; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; + +@Data +public class ReimportTask { + + private final EntityExportData data; + private final EntityImportSettings settings; + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 6db4655c59..addc8c35f2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -30,6 +30,4 @@ public class EntityImportSettings { private boolean saveAttributes; private boolean saveCredentials; - // internal - private boolean resetExternalIdsOfAnotherTenant; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java index b0ce419cac..cbc4ca1e02 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java @@ -19,10 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.BatchPreparedStatementSetter; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Repository; -import org.springframework.transaction.TransactionStatus; import org.springframework.transaction.annotation.Transactional; -import org.springframework.transaction.support.TransactionCallbackWithoutResult; -import org.springframework.transaction.support.TransactionTemplate; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.model.sql.RelationEntity; @@ -52,9 +49,6 @@ public class SqlRelationInsertRepository implements RelationInsertRepository { @Autowired protected JdbcTemplate jdbcTemplate; - @Autowired - private TransactionTemplate transactionTemplate; - protected Query getQuery(RelationEntity entity, String query) { Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class); if (entity.getAdditionalInfo() == null) { @@ -78,36 +72,31 @@ public class SqlRelationInsertRepository implements RelationInsertRepository { @Override public void saveOrUpdate(List entities) { - transactionTemplate.execute(new TransactionCallbackWithoutResult() { + jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { @Override - protected void doInTransactionWithoutResult(TransactionStatus status) { - jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() { - @Override - public void setValues(PreparedStatement ps, int i) throws SQLException { - RelationEntity relation = entities.get(i); - ps.setObject(1, relation.getFromId()); - ps.setString(2, relation.getFromType()); - ps.setObject(3, relation.getToId()); - ps.setString(4, relation.getToType()); - - ps.setString(5, relation.getRelationTypeGroup()); - ps.setString(6, relation.getRelationType()); - - if (relation.getAdditionalInfo() == null) { - ps.setString(7, null); - ps.setString(8, null); - } else { - String json = JacksonUtil.toString(relation.getAdditionalInfo()); - ps.setString(7, json); - ps.setString(8, json); - } - } + public void setValues(PreparedStatement ps, int i) throws SQLException { + RelationEntity relation = entities.get(i); + ps.setObject(1, relation.getFromId()); + ps.setString(2, relation.getFromType()); + ps.setObject(3, relation.getToId()); + ps.setString(4, relation.getToType()); + + ps.setString(5, relation.getRelationTypeGroup()); + ps.setString(6, relation.getRelationType()); + + if (relation.getAdditionalInfo() == null) { + ps.setString(7, null); + ps.setString(8, null); + } else { + String json = JacksonUtil.toString(relation.getAdditionalInfo()); + ps.setString(7, json); + ps.setString(8, json); + } + } - @Override - public int getBatchSize() { - return entities.size(); - } - }); + @Override + public int getBatchSize() { + return entities.size(); } }); }