From 82763495e00a0a3cfe2e79069e68104c229a2d88 Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Mon, 13 Jun 2022 16:21:05 +0300 Subject: [PATCH] Perofrmance improvements --- .../DefaultEntitiesExportImportService.java | 1 + .../impl/BaseEntityImportService.java | 38 +++++++++---------- .../impl/DashboardImportService.java | 5 +-- .../impl/RuleChainImportService.java | 5 +-- .../DefaultEntitiesVersionControlService.java | 28 ++++++++++---- .../sync/vc/data/EntitiesImportCtx.java | 7 +++- .../common/DefaultTbQueueRequestTemplate.java | 2 +- .../thingsboard/common/util/TbStopWatch.java | 13 ++++++- .../rule/engine/debug/TbMsgGeneratorNode.java | 2 +- 9 files changed, 64 insertions(+), 37 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index b69fb5906c..5ef7b84ce7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -96,6 +96,7 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS importResult.getSendEventsCallback().run(); } + ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); return importResult; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 7ba59bb3c4..f1cef3191c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -45,12 +45,10 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.security.model.SecurityUser; -import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @@ -83,13 +81,14 @@ public abstract class BaseEntityImportService importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { + EntityImportResult importResult = new EntityImportResult<>(); + IdProvider idProvider = new IdProvider(ctx, importResult); + E entity = exportData.getEntity(); - E existingEntity = findExistingEntity(ctx, entity); + E existingEntity = findExistingEntity(ctx, entity, idProvider); entity.setExternalId(entity.getId()); - EntityImportResult importResult = new EntityImportResult<>(); - IdProvider idProvider = new IdProvider(ctx, importResult); setOwner(ctx.getTenantId(), entity, idProvider); if (existingEntity == null) { entity.setId(null); @@ -125,39 +124,38 @@ public abstract class BaseEntityImportService relations, EntityImportResult importResult) { + private void importRelations(EntitiesImportCtx ctx, List relations, EntityImportResult importResult, IdProvider idProvider) { + var tenantId = ctx.getTenantId(); E entity = importResult.getSavedEntity(); importResult.addSaveReferencesCallback(() -> { for (EntityRelation relation : relations) { if (!relation.getTo().equals(entity.getId())) { - HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); - relation.setTo(to.getId()); + relation.setTo(idProvider.getInternalId(relation.getTo())); } if (!relation.getFrom().equals(entity.getId())) { - HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); - relation.setFrom(from.getId()); + relation.setFrom(idProvider.getInternalId(relation.getFrom())); } } if (importResult.getOldEntity() != null) { List existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); for (EntityRelation existingRelation : existingRelations) { if (!relations.contains(existingRelation)) { - relationService.deleteRelation(user.getTenantId(), existingRelation); + relationService.deleteRelation(tenantId, existingRelation); importResult.addSendEventsCallback(() -> { - entityActionService.logEntityAction(user, existingRelation.getFrom(), null, null, + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, ActionType.RELATION_DELETED, null, existingRelation); - entityActionService.logEntityAction(user, existingRelation.getTo(), null, null, + entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null, ActionType.RELATION_DELETED, null, existingRelation); }); } @@ -165,11 +163,11 @@ public abstract class BaseEntityImportService { - entityActionService.logEntityAction(user, relation.getFrom(), null, null, + entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, ActionType.RELATION_ADD_OR_UPDATE, null, relation); - entityActionService.logEntityAction(user, relation.getTo(), null, null, + entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, ActionType.RELATION_ADD_OR_UPDATE, null, relation); }); } @@ -223,7 +221,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(ctx.getTenantId(), entity.getId()))) .or(() -> { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java index c720f6d0c2..24238c78e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java @@ -30,7 +30,6 @@ import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; @@ -59,8 +58,8 @@ public class DashboardImportService extends BaseEntityImportService importResult = exportImportService.importEntity(ctx, entityData, true, true); return VersionLoadResult.success(EntityTypeLoadResult.builder() .entityType(importResult.getEntityType()) @@ -271,6 +272,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont List sendEventsCallbacks = new ArrayList<>(); EntitiesImportCtx ctx = new EntitiesImportCtx(user); + var sw = TbStopWatch.create("before"); List entityTypes = request.getEntityTypes().keySet().stream() .sorted(exportImportService.getEntityTypeComparatorForImport()).collect(Collectors.toList()); @@ -294,6 +296,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .findExistingByName(config.isFindExistingEntityByName()) .build()); for (EntityExportData entityData : entityDataList) { + sw.startNew("Entities " + entityType.name()); + log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); EntityImportResult importResult; try { importResult = exportImportService.importEntity(ctx, entityData, false, false); @@ -322,6 +326,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .build()); } + sw.startNew("Reimport"); toReimport.forEach((externalId, importSettings) -> { try { EntityExportData entityData = gitServiceQueue.getEntity(user.getTenantId(), request.getVersionId(), externalId).get(); @@ -341,6 +346,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } }); + sw.startNew("Remove Others"); request.getEntityTypes().keySet().stream() .filter(entityType -> request.getEntityTypes().get(entityType).isRemoveOtherEntities()) .sorted(exportImportService.getEntityTypeComparatorForImport().reversed()) @@ -361,6 +367,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }); }); + sw.startNew("Callbacks"); + for (ThrowingRunnable saveReferencesCallback : saveReferencesCallbacks) { try { saveReferencesCallback.run(); @@ -375,6 +383,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont log.error("Failed to send events for entity", e); } } + + sw.stop(); + for (var task : sw.getTaskInfo()) { + log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis()); + } + log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis()); return VersionLoadResult.success(new ArrayList<>(results.values())); } @@ -412,8 +426,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .exportCredentials(otherVersion.hasCredentials()) .build()); return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), - JacksonUtil.toPrettyString(currentVersion.sort()), - JacksonUtil.toPrettyString(otherVersion.sort())), + JacksonUtil.toPrettyString(currentVersion.sort()), + JacksonUtil.toPrettyString(otherVersion.sort())), rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); }, MoreExecutors.directExecutor()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index 4fbdb28257..93de254d2a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.sync.vc.data; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; @@ -24,6 +25,7 @@ import org.thingsboard.server.service.security.model.SecurityUser; import java.util.HashMap; import java.util.Map; +@Slf4j @Data public class EntitiesImportCtx { @@ -66,10 +68,13 @@ public class EntitiesImportCtx { } public EntityId getInternalId(EntityId externalId) { - return externalToInternalIdMap.get(externalId); + var result = externalToInternalIdMap.get(externalId); + log.debug("[{}][{}] Local cache {} for id", externalId.getEntityType(), externalId.getId(), result != null ? "hit" : "miss"); + return result; } public void putInternalId(EntityId externalId, EntityId internalId) { + log.debug("[{}][{}] Local cache put: {}", externalId.getEntityType(), externalId.getId(), internalId); externalToInternalIdMap.put(externalId, internalId); } } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java index e38677b2ce..9fbe9b1104 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/common/DefaultTbQueueRequestTemplate.java @@ -96,7 +96,7 @@ public class DefaultTbQueueRequestTemplate { log.trace("onMsg onSuccess callback, took {}ms, config {}, msg {}", sw.stopAndGetTotalTimeMillis(), config, msg);