Perofrmance improvements

4 years ago · 82763495e0
9 changed files with 64 additions and 37 deletions
--- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
@ -96,6 +96,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
            importResult.getSendEventsCallback().run();
        }

+        ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId());
        return importResult;
    }

--- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
@ -45,12 +45,10 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
 import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
 import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
-import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
 import org.thingsboard.server.dao.relation.RelationService;
 import org.thingsboard.server.service.action.EntityActionService;
 import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
 import org.thingsboard.server.service.security.model.SecurityUser;
-import org.thingsboard.server.service.security.permission.Operation;
 import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
 import org.thingsboard.server.service.sync.ie.importing.EntityImportService;
 import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@ -83,13 +81,14 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
    @Transactional(rollbackFor = Exception.class)
    @Override
    public EntityImportResult<E> importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException {
+        EntityImportResult<E> importResult = new EntityImportResult<>();
+        IdProvider idProvider = new IdProvider(ctx, importResult);
+
        E entity = exportData.getEntity();
-        E existingEntity = findExistingEntity(ctx, entity);
+        E existingEntity = findExistingEntity(ctx, entity, idProvider);

        entity.setExternalId(entity.getId());

-        EntityImportResult<E> importResult = new EntityImportResult<>();
-        IdProvider idProvider = new IdProvider(ctx, importResult);
        setOwner(ctx.getTenantId(), entity, idProvider);
        if (existingEntity == null) {
            entity.setId(null);
@ -125,39 +124,38 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
        });

        if (ctx.isUpdateRelations() && exportData.getRelations() != null) {
-            importRelations(ctx.getUser(), exportData.getRelations(), importResult);
+            importRelations(ctx, exportData.getRelations(), importResult, idProvider);
        }
        if (ctx.isSaveAttributes() && exportData.getAttributes() != null) {
            importAttributes(ctx.getUser(), exportData.getAttributes(), importResult);
        }
    }

-    private void importRelations(SecurityUser user, List<EntityRelation> relations, EntityImportResult<E> importResult) {
+    private void importRelations(EntitiesImportCtx ctx, List<EntityRelation> relations, EntityImportResult<E> importResult, IdProvider idProvider) {
+        var tenantId = ctx.getTenantId();
        E entity = importResult.getSavedEntity();
        importResult.addSaveReferencesCallback(() -> {
            for (EntityRelation relation : relations) {
                if (!relation.getTo().equals(entity.getId())) {
-                    HasId<EntityId> to = findInternalEntity(user.getTenantId(), relation.getTo());
-                    relation.setTo(to.getId());
+                    relation.setTo(idProvider.getInternalId(relation.getTo()));
                }
                if (!relation.getFrom().equals(entity.getId())) {
-                    HasId<EntityId> from = findInternalEntity(user.getTenantId(), relation.getFrom());
-                    relation.setFrom(from.getId());
+                    relation.setFrom(idProvider.getInternalId(relation.getFrom()));
                }
            }

            if (importResult.getOldEntity() != null) {
                List<EntityRelation> existingRelations = new ArrayList<>();
-                existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
-                existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
+                existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON));
+                existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON));

                for (EntityRelation existingRelation : existingRelations) {
                    if (!relations.contains(existingRelation)) {
-                        relationService.deleteRelation(user.getTenantId(), existingRelation);
+                        relationService.deleteRelation(tenantId, existingRelation);
                        importResult.addSendEventsCallback(() -> {
-                            entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null,
+                            entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null,
                                    ActionType.RELATION_DELETED, null, existingRelation);
-                            entityActionService.logEntityAction(user, existingRelation.getTo(), null, null,
+                            entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null,
                                    ActionType.RELATION_DELETED, null, existingRelation);
                        });
                    }
@ -165,11 +163,11 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
            }

            for (EntityRelation relation : relations) {
-                relationService.saveRelation(user.getTenantId(), relation);
+                relationService.saveRelation(tenantId, relation);
                importResult.addSendEventsCallback(() -> {
-                    entityActionService.logEntityAction(user, relation.getFrom(), null, null,
+                    entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null,
                            ActionType.RELATION_ADD_OR_UPDATE, null, relation);
-                    entityActionService.logEntityAction(user, relation.getTo(), null, null,
+                    entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null,
                            ActionType.RELATION_ADD_OR_UPDATE, null, relation);
                });
            }
@ -223,7 +221,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo


    @SuppressWarnings("unchecked")
-    protected E findExistingEntity(EntitiesImportCtx ctx, E entity) {
+    protected E findExistingEntity(EntitiesImportCtx ctx, E entity, IdProvider idProvider) {
        return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(ctx.getTenantId(), entity.getId()))
                .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId())))
                .or(() -> {
--- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.id.CustomerId;
 import org.thingsboard.server.common.data.id.DashboardId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.sync.ie.EntityExportData;
-import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
 import org.thingsboard.server.dao.dashboard.DashboardService;
 import org.thingsboard.server.queue.util.TbCoreComponent;
 import org.thingsboard.server.service.security.model.SecurityUser;
@ -59,8 +58,8 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
    }

    @Override
-    protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard) {
-        Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard);
+    protected Dashboard findExistingEntity(EntitiesImportCtx ctx, Dashboard dashboard, IdProvider idProvider) {
+        Dashboard existingDashboard = super.findExistingEntity(ctx, dashboard, idProvider);
        if (existingDashboard == null && ctx.isFindExistingByName()) {
            existingDashboard = dashboardService.findTenantDashboardsByTitle(ctx.getTenantId(), dashboard.getName()).stream().findFirst().orElse(null);
        }
--- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
@ -28,7 +28,6 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
 import org.thingsboard.server.common.data.rule.RuleChain;
 import org.thingsboard.server.common.data.rule.RuleChainMetaData;
 import org.thingsboard.server.common.data.rule.RuleChainType;
-import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
 import org.thingsboard.server.common.data.sync.ie.RuleChainExportData;
 import org.thingsboard.server.dao.rule.RuleChainService;
 import org.thingsboard.server.queue.util.TbCoreComponent;
@ -53,8 +52,8 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
    }

    @Override
-    protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain) {
-        RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain);
+    protected RuleChain findExistingEntity(EntitiesImportCtx ctx, RuleChain ruleChain, IdProvider idProvider) {
+        RuleChain existingRuleChain = super.findExistingEntity(ctx, ruleChain, idProvider);
        if (existingRuleChain == null && ctx.isFindExistingByName()) {
            existingRuleChain = ruleChainService.findTenantRuleChainsByTypeAndName(ctx.getTenantId(), ruleChain.getType(), ruleChain.getName()).stream().findFirst().orElse(null);
        }
--- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
@ -27,6 +27,7 @@ import org.springframework.stereotype.Service;
 import org.springframework.transaction.support.TransactionCallback;
 import org.springframework.transaction.support.TransactionTemplate;
 import org.thingsboard.common.util.JacksonUtil;
+import org.thingsboard.common.util.TbStopWatch;
 import org.thingsboard.common.util.ThingsBoardExecutors;
 import org.thingsboard.server.common.data.EntityType;
 import org.thingsboard.server.common.data.ExportableEntity;
@ -246,11 +247,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
    private VersionLoadResult loadSingleEntity(SecurityUser user, VersionLoadConfig config, EntityExportData entityData) {
        try {
            var ctx = new EntitiesImportCtx(user, EntityImportSettings.builder()
-                            .updateRelations(config.isLoadRelations())
-                            .saveAttributes(config.isLoadAttributes())
-                            .saveCredentials(config.isLoadCredentials())
-                            .findExistingByName(false)
-                            .build());
+                    .updateRelations(config.isLoadRelations())
+                    .saveAttributes(config.isLoadAttributes())
+                    .saveCredentials(config.isLoadCredentials())
+                    .findExistingByName(false)
+                    .build());
            EntityImportResult<?> importResult = exportImportService.importEntity(ctx, entityData, true, true);
            return VersionLoadResult.success(EntityTypeLoadResult.builder()
                    .entityType(importResult.getEntityType())
@ -271,6 +272,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
        List<ThrowingRunnable> sendEventsCallbacks = new ArrayList<>();

        EntitiesImportCtx ctx = new EntitiesImportCtx(user);
+        var sw = TbStopWatch.create("before");

        List<EntityType> entityTypes = request.getEntityTypes().keySet().stream()
                .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList());
@ -294,6 +296,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                        .findExistingByName(config.isFindExistingEntityByName())
                        .build());
                for (EntityExportData entityData : entityDataList) {
+                    sw.startNew("Entities " + entityType.name());
+                    log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
                    EntityImportResult<?> importResult;
                    try {
                        importResult = exportImportService.importEntity(ctx, entityData, false, false);
@ -322,6 +326,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                    .build());
        }

+        sw.startNew("Reimport");
        toReimport.forEach((externalId, importSettings) -> {
            try {
                EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get();
@ -341,6 +346,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
            }
        });

+        sw.startNew("Remove Others");
        request.getEntityTypes().keySet().stream()
                .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities())
                .sorted(exportImportService.getEntityTypeComparatorForImport().reversed())
@ -361,6 +367,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                    });
                });

+        sw.startNew("Callbacks");
+
        for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) {
            try {
                saveReferencesCallback.run();
@ -375,6 +383,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                log.error("Failed to send events for entity", e);
            }
        }
+
+        sw.stop();
+        for (var task : sw.getTaskInfo()) {
+            log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
+        }
+        log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
        return VersionLoadResult.success(new ArrayList<>(results.values()));
    }

@ -412,8 +426,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
                            .exportCredentials(otherVersion.hasCredentials())
                            .build());
                    return transform(gitServiceQueue.getContentsDiff(user.getTenantId(),
-                                    JacksonUtil.toPrettyString(currentVersion.sort()),
-                                    JacksonUtil.toPrettyString(otherVersion.sort())),
+                            JacksonUtil.toPrettyString(currentVersion.sort()),
+                            JacksonUtil.toPrettyString(otherVersion.sort())),
                            rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor());
                }, MoreExecutors.directExecutor());
    }
--- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java
+++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java
@ -16,6 +16,7 @@
 package org.thingsboard.server.service.sync.vc.data;

 import lombok.Data;
+import lombok.extern.slf4j.Slf4j;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
 import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
@ -24,6 +25,7 @@ import org.thingsboard.server.service.security.model.SecurityUser;
 import java.util.HashMap;
 import java.util.Map;

+@Slf4j
@Data
 public class EntitiesImportCtx {

@ -66,10 +68,13 @@ public class EntitiesImportCtx {
    }

    public EntityId getInternalId(EntityId externalId) {
-        return externalToInternalIdMap.get(externalId);
+        var result = externalToInternalIdMap.get(externalId);
+        log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss");
+        return result;
    }

    public void putInternalId(EntityId externalId, EntityId internalId) {
+        log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId);
        externalToInternalIdMap.put(externalId, internalId);
    }
 }
--- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java
+++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java
@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate<Request extends TbQueueMsg, Response

    void mainLoop() {
        while (!stopped) {
-            TbStopWatch sw = TbStopWatch.startNew();
+            TbStopWatch sw = TbStopWatch.create();
            try {
                fetchAndProcessResponses();
            } catch (Throwable e) {
--- a/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java
+++ b/common/util/src/main/java/org/thingsboard/common/util/TbStopWatch.java
@ -28,12 +28,23 @@ import org.springframework.util.StopWatch;
 * */
 public class TbStopWatch extends StopWatch {

-    public static TbStopWatch startNew(){
+    public static TbStopWatch create(){
        TbStopWatch stopWatch = new TbStopWatch();
        stopWatch.start();
        return stopWatch;
    }

+    public static TbStopWatch create(String taskName){
+        TbStopWatch stopWatch = new TbStopWatch();
+        stopWatch.start(taskName);
+        return stopWatch;
+    }
+
+    public void startNew(String taskName){
+        stop();
+        start(taskName);
+    }
+
    public long stopAndGetTotalTimeMillis(){
        stop();
        return getTotalTimeMillis();
--- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java
+++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/debug/TbMsgGeneratorNode.java
@ -105,7 +105,7 @@ public class TbMsgGeneratorNode implements TbNode {
    public void onMsg(TbContext ctx, TbMsg msg) {
        log.trace("onMsg, config {}, msg {}", config, msg);
        if (initialized.get() && msg.getType().equals(TB_MSG_GENERATOR_NODE_MSG) && msg.getId().equals(nextTickId)) {
-            TbStopWatch sw = TbStopWatch.startNew();
+            TbStopWatch sw = TbStopWatch.create();
            withCallback(generate(ctx, msg),
                    m -> {
                        log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg);