Backport of the permance improvements from PE

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@@ -60,6 +60,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.UUID;
 import java.util.stream.Collectors;
 
@@ -280,46 +281,59 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
             return entity.getId();
         }
 
-        public Optional<EntityId> getInternalIdByUuid(UUID externalUuid) {
+        public Optional<EntityId> getInternalIdByUuid(UUID externalUuid, boolean fetchAllUUIDs, Set<EntityType> hints) {
             if (externalUuid.equals(EntityId.NULL_UUID)) return Optional.empty();
 
             for (EntityType entityType : EntityType.values()) {
-                EntityId externalId;
+                Optional<EntityId> externalIdOpt = buildEntityId(entityType, externalUuid);
-                try {
+                if (!externalIdOpt.isPresent()) {
-                    externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid);
-                } catch (Exception e) {
                     continue;
                 }
-                EntityId internalId = ctx.getInternalId(externalId);
+                EntityId internalId = ctx.getInternalId(externalIdOpt.get());
                 if (internalId != null) {
                     return Optional.of(internalId);
                 }
             }
 
+            if (fetchAllUUIDs) {
+                for (EntityType entityType : hints) {
+                    Optional<EntityId> internalId = lookupInDb(externalUuid, entityType);
+                    if (internalId.isPresent()) return internalId;
+                }
                 for (EntityType entityType : EntityType.values()) {
-                EntityId externalId;
+                    if (hints.contains(entityType)) {
-                try {
-                    externalId = EntityIdFactory.getByTypeAndUuid(entityType, externalUuid);
-                } catch (Exception e) {
                         continue;
                     }
+                    Optional<EntityId> internalId = lookupInDb(externalUuid, entityType);
+                    if (internalId.isPresent()) return internalId;
+                }
+            }
 
-                EntityId internalId = getInternalId(externalId, false);
+            importResult.setUpdatedAllExternalIds(false);
-                if (internalId != null) {
+            return Optional.empty();
-                    return Optional.of(internalId);
-                } else if (ctx.isResetExternalIdsOfAnotherTenant()) {
-                    try {
-                        if (exportableEntitiesService.findEntityById(externalId) != null) {
-                            return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, EntityId.NULL_UUID));
         }
-                    } catch (Exception ignored) {
+
+        private Optional<EntityId> lookupInDb(UUID externalUuid, EntityType entityType) {
+            Optional<EntityId> externalIdOpt = buildEntityId(entityType, externalUuid);
+            if (externalIdOpt.isEmpty() || ctx.isNotFound(externalIdOpt.get())) {
+                return Optional.empty();
             }
+            EntityId internalId = getInternalId(externalIdOpt.get(), false);
+            if (internalId != null) {
+                return Optional.of(internalId);
+            } else {
+                ctx.registerNotFound(externalIdOpt.get());
             }
+            return Optional.empty();
         }
 
-            importResult.setUpdatedAllExternalIds(false);
+        private Optional<EntityId> buildEntityId(EntityType entityType, UUID externalUuid) {
+            try {
+                return Optional.of(EntityIdFactory.getByTypeAndUuid(entityType, externalUuid));
+            } catch (Exception e) {
                 return Optional.empty();
             }
+        }
 
     }
 

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java

@@ -37,8 +37,10 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
 import org.thingsboard.server.utils.RegexUtils;
 
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
 import java.util.Optional;
 import java.util.Set;
 import java.util.UUID;
@@ -49,6 +51,8 @@ import java.util.stream.Collectors;
 @RequiredArgsConstructor
 public class DashboardImportService extends BaseEntityImportService<DashboardId, Dashboard, EntityExportData<Dashboard>> {
 
+    private static final LinkedHashSet<EntityType> HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.DASHBOARD, EntityType.DEVICE, EntityType.ASSET));
+
     private final DashboardService dashboardService;
 
 
@@ -78,7 +82,7 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
                     if (field.equals("id")) continue;
                     JsonNode oldFieldValue = entityAlias.get(field);
                     JsonNode newFieldValue = JacksonUtil.toJsonNode(RegexUtils.replace(oldFieldValue.toString(), RegexUtils.UUID_PATTERN, uuid -> {
-                        return idProvider.getInternalIdByUuid(UUID.fromString(uuid))
+                        return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS)
                                 .map(entityId -> entityId.getId().toString()).orElse(uuid);
                     }));
                     ((ObjectNode) entityAlias).set(field, newFieldValue);

application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@@ -35,7 +35,9 @@ import org.thingsboard.server.service.security.model.SecurityUser;
 import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
 import org.thingsboard.server.utils.RegexUtils;
 
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedHashSet;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -44,6 +46,8 @@ import java.util.UUID;
 @RequiredArgsConstructor
 public class RuleChainImportService extends BaseEntityImportService<RuleChainId, RuleChain, RuleChainExportData> {
 
+    private static final LinkedHashSet<EntityType> HINTS = new LinkedHashSet<>(Arrays.asList(EntityType.RULE_CHAIN, EntityType.DEVICE, EntityType.ASSET));
+
     private final RuleChainService ruleChainService;
 
     @Override
@@ -70,7 +74,7 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
 
                     JsonNode ruleNodeConfig = ruleNode.getConfiguration();
                     String newRuleNodeConfigJson = RegexUtils.replace(ruleNodeConfig.toString(), RegexUtils.UUID_PATTERN, uuid -> {
-                        return idProvider.getInternalIdByUuid(UUID.fromString(uuid))
+                        return idProvider.getInternalIdByUuid(UUID.fromString(uuid), ctx.isFetchAllUUIDs(), HINTS)
                                 .map(entityId -> entityId.getId().toString())
                                 .orElse(uuid);
                     });

application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@@ -77,6 +77,7 @@ import org.thingsboard.server.service.sync.vc.data.ComplexEntitiesExportCtx;
 import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
 import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
 import org.thingsboard.server.service.sync.vc.data.EntityTypeExportCtx;
+import org.thingsboard.server.service.sync.vc.data.ReimportTask;
 import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx;
 import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService;
 
@@ -282,8 +283,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
         for (EntityType entityType : entityTypes) {
             log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
             sw.startNew("Entities " + entityType.name());
-            EntityImportSettings settings = getEntityImportSettings(request, entityType);
+            ctx.setSettings(getEntityImportSettings(request, entityType));
-            ctx.setSettings(settings);
             importEntities(ctx, entityType);
         }
 
@@ -337,7 +337,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                     throw new LoadEntityException(entityData, e);
                 }
                 if (importResult.getUpdatedAllExternalIds() != null && !importResult.getUpdatedAllExternalIds()) {
-                    ctx.getToReimport().put(entityData.getEntity().getExternalId(), ctx.getSettings());
+                    ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(entityData, ctx.getSettings()));
                     continue;
                 }
 
@@ -351,11 +351,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
 
     @SuppressWarnings({"rawtypes", "unchecked"})
     private void reimport(EntitiesImportCtx ctx) {
-        ctx.getToReimport().forEach((externalId, importSettings) -> {
+        ctx.setFetchAllUUIDs(true);
+        ctx.getToReimport().forEach((externalId, task) -> {
             try {
-                EntityExportData entityData = gitServiceQueue.getEntity(ctx.getTenantId(), ctx.getVersionId(), externalId).get();
+                EntityExportData entityData = task.getData();
-                importSettings.setResetExternalIdsOfAnotherTenant(true);
+                var settings = task.getSettings();
-                ctx.setSettings(importSettings);
+                ctx.setSettings(settings);
                 EntityImportResult<?> importResult = exportImportService.importEntity(ctx, entityData);
 
                 ctx.registerResult(externalId.getEntityType(), importResult.getOldEntity() == null);

application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java

@@ -29,6 +29,7 @@ import org.thingsboard.server.service.security.model.SecurityUser;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -43,14 +44,15 @@ public class EntitiesImportCtx {
 
     private final Map<EntityType, EntityTypeLoadResult> results = new HashMap<>();
     private final Map<EntityType, Set<EntityId>> importedEntities = new HashMap<>();
-    private final Map<EntityId, EntityImportSettings> toReimport = new HashMap<>();
+    private final Map<EntityId, ReimportTask> toReimport = new HashMap<>();
     private final List<ThrowingRunnable> referenceCallbacks = new ArrayList<>();
     private final List<ThrowingRunnable> eventCallbacks = new ArrayList<>();
-
     private final Map<EntityId, EntityId> externalToInternalIdMap = new HashMap<>();
+    private final Set<EntityId> notFoundIds = new HashSet<>();
 
     private final Set<EntityRelation> relations = new LinkedHashSet<>();
 
+    private boolean fetchAllUUIDs = false;
     private EntityImportSettings settings;
 
     public EntitiesImportCtx(SecurityUser user, String versionId) {
@@ -83,10 +85,6 @@ public class EntitiesImportCtx {
         return getSettings().isSaveCredentials();
     }
 
-    public boolean isResetExternalIdsOfAnotherTenant() {
-        return getSettings().isResetExternalIdsOfAnotherTenant();
-    }
-
     public EntityId getInternalId(EntityId externalId) {
         var result = externalToInternalIdMap.get(externalId);
         log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss");
@@ -128,5 +126,12 @@ public class EntitiesImportCtx {
         }
     }
 
+    public void registerNotFound(EntityId externalId) {
+        notFoundIds.add(externalId);
+    }
+
+    public boolean isNotFound(EntityId externalId) {
+        return notFoundIds.contains(externalId);
+    }
 
 }

application/src/main/java/org/thingsboard/server/service/sync/vc/data/ReimportTask.java

@@ -0,0 +1,28 @@
+/**
+ * Copyright © 2016-2022 The Thingsboard Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.thingsboard.server.service.sync.vc.data;
+
+import lombok.Data;
+import org.thingsboard.server.common.data.sync.ie.EntityExportData;
+import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
+
+@Data
+public class ReimportTask {
+
+    private final EntityExportData data;
+    private final EntityImportSettings settings;
+
+}

common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java

@@ -30,6 +30,4 @@ public class EntityImportSettings {
     private boolean saveAttributes;
     private boolean saveCredentials;
 
-    // internal
-    private boolean resetExternalIdsOfAnotherTenant;
 }

dao/src/main/java/org/thingsboard/server/dao/sql/relation/SqlRelationInsertRepository.java

@@ -19,10 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.jdbc.core.BatchPreparedStatementSetter;
 import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.stereotype.Repository;
-import org.springframework.transaction.TransactionStatus;
 import org.springframework.transaction.annotation.Transactional;
-import org.springframework.transaction.support.TransactionCallbackWithoutResult;
-import org.springframework.transaction.support.TransactionTemplate;
 import org.thingsboard.common.util.JacksonUtil;
 import org.thingsboard.server.dao.model.sql.RelationEntity;
 
@@ -52,9 +49,6 @@ public class SqlRelationInsertRepository implements RelationInsertRepository {
     @Autowired
     protected JdbcTemplate jdbcTemplate;
 
-    @Autowired
-    private TransactionTemplate transactionTemplate;
-
     protected Query getQuery(RelationEntity entity, String query) {
         Query nativeQuery = entityManager.createNativeQuery(query, RelationEntity.class);
         if (entity.getAdditionalInfo() == null) {
@@ -78,9 +72,6 @@ public class SqlRelationInsertRepository implements RelationInsertRepository {
 
     @Override
     public void saveOrUpdate(List<RelationEntity> entities) {
-        transactionTemplate.execute(new TransactionCallbackWithoutResult() {
-            @Override
-            protected void doInTransactionWithoutResult(TransactionStatus status) {
         jdbcTemplate.batchUpdate(INSERT_ON_CONFLICT_DO_UPDATE_JDBC, new BatchPreparedStatementSetter() {
             @Override
             public void setValues(PreparedStatement ps, int i) throws SQLException {
@@ -109,7 +100,5 @@ public class SqlRelationInsertRepository implements RelationInsertRepository {
             }
         });
     }
-        });
-    }
 
 }