From 15c1e215d44cefed77ffaa4567f977fa1fa2a66e Mon Sep 17 00:00:00 2001 From: Andrii Shvaika Date: Thu, 16 Jun 2022 12:09:17 +0300 Subject: [PATCH] Improve performance of relation and rule chain service --- .../processing/AbstractConsumerService.java | 2 +- .../impl/BaseEntityImportService.java | 13 ++++- .../service/DefaultTransportService.java | 2 +- .../dao/relation/BaseRelationService.java | 50 +++++++------------ .../server/dao/relation/RelationDao.java | 8 ++- .../server/dao/rule/BaseRuleChainService.java | 29 ++++++++--- .../server/dao/rule/RuleNodeDao.java | 3 ++ .../dao/sql/relation/JpaRelationDao.java | 48 ++++++++++++++---- .../dao/sql/relation/RelationRepository.java | 11 ++++ .../server/dao/sql/rule/JpaRuleNodeDao.java | 9 +++- .../dao/sql/rule/RuleNodeRepository.java | 2 + 11 files changed, 121 insertions(+), 56 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index f1be88a44d..0c130c9f62 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -160,7 +160,7 @@ public abstract class AbstractConsumerService importEntity(EntitiesImportCtx ctx, D exportData) throws ThingsboardException { +// TbStopWatch sw = TbStopWatch.create("find"); EntityImportResult importResult = new EntityImportResult<>(); IdProvider idProvider = new IdProvider(ctx, importResult); @@ -100,16 +103,22 @@ public abstract class BaseEntityImportService inboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - inboundRelations.addAll(relationDao.findAllByTo(tenantId, entityId, typeGroup)); - } - - List outboundRelations = new ArrayList<>(); - for (RelationTypeGroup typeGroup : RelationTypeGroup.values()) { - outboundRelations.addAll(relationDao.findAllByFrom(tenantId, entityId, typeGroup)); - } + List inboundRelations = new ArrayList<>(relationDao.findAllByTo(tenantId, entityId)); + List outboundRelations = new ArrayList<>(relationDao.findAllByFrom(tenantId, entityId)); - for (EntityRelation relation : inboundRelations) { - delete(tenantId, relation, true); - } + if (!inboundRelations.isEmpty()) { + try { + relationDao.deleteInboundRelations(tenantId, entityId); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", inboundRelations, e); + } - for (EntityRelation relation : outboundRelations) { - delete(tenantId, relation, false); + for (EntityRelation relation : inboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } } - relationDao.deleteOutboundRelations(tenantId, entityId); + if (!outboundRelations.isEmpty()) { + relationDao.deleteOutboundRelations(tenantId, entityId); + for (EntityRelation relation : outboundRelations) { + eventPublisher.publishEvent(EntityRelationEvent.from(relation)); + } + } } @Override @@ -269,18 +267,6 @@ public class BaseRelationService implements RelationService { } } - boolean delete(TenantId tenantId, EntityRelation relation, boolean deleteFromDb) { - eventPublisher.publishEvent(EntityRelationEvent.from(relation)); - if (deleteFromDb) { - try { - return relationDao.deleteRelation(tenantId, relation); - } catch (ConcurrencyFailureException e) { - log.debug("Concurrency exception while deleting relations [{}]", relation, e); - } - } - return false; - } - @Override public List findByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup) { validate(from); diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java index 9ed3a5672f..180996a6e8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationDao.java @@ -32,10 +32,14 @@ public interface RelationDao { List findAllByFrom(TenantId tenantId, EntityId from, RelationTypeGroup typeGroup); + List findAllByFrom(TenantId tenantId, EntityId from); + List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup); List findAllByTo(TenantId tenantId, EntityId to, RelationTypeGroup typeGroup); + List findAllByTo(TenantId tenantId, EntityId to); + List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup); ListenableFuture checkRelation(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); @@ -56,7 +60,9 @@ public interface RelationDao { ListenableFuture deleteRelationAsync(TenantId tenantId, EntityId from, EntityId to, String relationType, RelationTypeGroup typeGroup); - boolean deleteOutboundRelations(TenantId tenantId, EntityId entity); + void deleteOutboundRelations(TenantId tenantId, EntityId entity); + + void deleteInboundRelations(TenantId tenantId, EntityId entity); ListenableFuture deleteOutboundRelationsAsync(TenantId tenantId, EntityId entity); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 03b3886cfc..f785eae0b7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -141,6 +141,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC List nodes = ruleChainMetaData.getNodes(); List toAddOrUpdate = new ArrayList<>(); List toDelete = new ArrayList<>(); + List relations = new ArrayList<>(); Map ruleNodeIndexMap = new HashMap<>(); if (nodes != null) { @@ -171,15 +172,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC for (RuleNode node : toAddOrUpdate) { node.setRuleChainId(ruleChain.getId()); RuleNode savedNode = ruleNodeDao.save(tenantId, node); - createRelation(tenantId, new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), + relations.add(new EntityRelation(ruleChainMetaData.getRuleChainId(), savedNode.getId(), EntityRelation.CONTAINS_TYPE, RelationTypeGroup.RULE_CHAIN)); int index = nodes.indexOf(node); nodes.set(index, savedNode); ruleNodeIndexMap.put(savedNode.getId(), index); } } - for (RuleNode node : toDelete) { - deleteRuleNode(tenantId, node.getId()); + if (!toDelete.isEmpty()) { + deleteRuleNodes(tenantId, toDelete); } RuleNodeId firstRuleNodeId = null; if (nodes != null) { @@ -196,7 +197,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC EntityId from = nodes.get(nodeConnection.getFromIndex()).getId(); EntityId to = nodes.get(nodeConnection.getToIndex()).getId(); String type = nodeConnection.getType(); - createRelation(tenantId, new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); + relations.add(new EntityRelation(from, to, type, RelationTypeGroup.RULE_NODE)); } } if (ruleChainMetaData.getRuleChainConnections() != null) { @@ -222,7 +223,7 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleChainToRuleNode.setTo(targetNode.getId()); sourceRuleChainToRuleNode.setType(EntityRelation.CONTAINS_TYPE); sourceRuleChainToRuleNode.setTypeGroup(RelationTypeGroup.RULE_CHAIN); - relationService.saveRelation(tenantId, sourceRuleChainToRuleNode); + relations.add(sourceRuleChainToRuleNode); EntityRelation sourceRuleNodeToTargetRuleNode = new EntityRelation(); EntityId from = nodes.get(nodeToRuleChainConnection.getFromIndex()).getId(); @@ -230,11 +231,15 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC sourceRuleNodeToTargetRuleNode.setTo(targetNode.getId()); sourceRuleNodeToTargetRuleNode.setType(nodeToRuleChainConnection.getType()); sourceRuleNodeToTargetRuleNode.setTypeGroup(RelationTypeGroup.RULE_NODE); - relationService.saveRelation(tenantId, sourceRuleNodeToTargetRuleNode); - } + relations.add(sourceRuleNodeToTargetRuleNode); + } } } + if (!relations.isEmpty()) { + relationService.saveRelations(tenantId, relations); + } + return RuleChainUpdateResult.successful(updatedRuleNodes); } @@ -710,6 +715,14 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC deleteRuleNodes(tenantId, ruleChainId); } + private void deleteRuleNodes(TenantId tenantId, List ruleNodes) { + List ruleNodeIds = ruleNodes.stream().map(RuleNode::getId).collect(Collectors.toList()); + for (var node : ruleNodes) { + deleteEntityRelations(tenantId, node.getId()); + } + ruleNodeDao.deleteByIdIn(ruleNodeIds); + } + @Override public void deleteRuleNodes(TenantId tenantId, RuleChainId ruleChainId) { List nodeRelations = getRuleChainToNodeRelations(tenantId, ruleChainId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java index b4f087b98a..bd3efcd205 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/RuleNodeDao.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.rule; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -31,4 +32,6 @@ public interface RuleNodeDao extends Dao { List findRuleNodesByTenantIdAndType(TenantId tenantId, String type, String search); PageData findAllRuleNodesByType(String type, PageLink pageLink); + + void deleteByIdIn(List ruleNodeIds); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java index 550197c924..5b8a1a6e5d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/JpaRelationDao.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -33,6 +33,8 @@ import org.thingsboard.server.dao.model.sql.RelationEntity; import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.dao.sql.JpaAbstractDaoListeningExecutorService; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.stream.Collectors; @@ -44,6 +46,12 @@ import java.util.stream.Collectors; @Component public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService implements RelationDao { + private static final List ALL_TYPE_GROUP_NAMES = new ArrayList<>(); + + static { + Arrays.stream(RelationTypeGroup.values()).map(RelationTypeGroup::name).forEach(ALL_TYPE_GROUP_NAMES::add); + } + @Autowired private RelationRepository relationRepository; @@ -59,6 +67,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByFrom(TenantId tenantId, EntityId from) { + return DaoUtil.convertDataList( + relationRepository.findAllByFromIdAndFromTypeAndRelationTypeGroupIn( + from.getId(), + from.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByFromAndType(TenantId tenantId, EntityId from, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -78,6 +95,15 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple typeGroup.name())); } + @Override + public List findAllByTo(TenantId tenantId, EntityId to) { + return DaoUtil.convertDataList( + relationRepository.findAllByToIdAndToTypeAndRelationTypeGroupIn( + to.getId(), + to.getEntityType().name(), + ALL_TYPE_GROUP_NAMES)); + } + @Override public List findAllByToAndType(TenantId tenantId, EntityId to, String relationType, RelationTypeGroup typeGroup) { return DaoUtil.convertDataList( @@ -164,19 +190,21 @@ public class JpaRelationDao extends JpaAbstractDaoListeningExecutorService imple } @Override - public boolean deleteOutboundRelations(TenantId tenantId, EntityId entity) { - boolean relationExistsBeforeDelete = false; + public void deleteOutboundRelations(TenantId tenantId, EntityId entity) { try { - relationExistsBeforeDelete = relationRepository - .findAllByFromIdAndFromType(entity.getId(), entity.getEntityType().name()) - .size() > 0; - if (relationExistsBeforeDelete) { - relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); - } + relationRepository.deleteByFromIdAndFromType(entity.getId(), entity.getEntityType().name()); + } catch (ConcurrencyFailureException e) { + log.debug("Concurrency exception while deleting relations [{}]", entity, e); + } + } + + @Override + public void deleteInboundRelations(TenantId tenantId, EntityId entity) { + try { + relationRepository.deleteByToIdAndToTypeAndRelationTypeGroupIn(entity.getId(), entity.getEntityType().name(), ALL_TYPE_GROUP_NAMES); } catch (ConcurrencyFailureException e) { log.debug("Concurrency exception while deleting relations [{}]", entity, e); } - return relationExistsBeforeDelete; } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java index bddd344672..1f776fd44d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/relation/RelationRepository.java @@ -35,6 +35,10 @@ public interface RelationRepository String fromType, String relationTypeGroup); + List findAllByFromIdAndFromTypeAndRelationTypeGroupIn(UUID fromId, + String fromType, + List relationTypeGroups); + List findAllByFromIdAndFromTypeAndRelationTypeAndRelationTypeGroup(UUID fromId, String fromType, String relationType, @@ -44,6 +48,10 @@ public interface RelationRepository String toType, String relationTypeGroup); + List findAllByToIdAndToTypeAndRelationTypeGroupIn(UUID toId, + String toType, + List relationTypeGroups); + List findAllByToIdAndToTypeAndRelationTypeAndRelationTypeGroup(UUID toId, String toType, String relationType, @@ -66,4 +74,7 @@ public interface RelationRepository @Transactional void deleteByFromIdAndFromType(UUID fromId, String fromType); + @Transactional + void deleteByToIdAndToTypeAndRelationTypeGroupIn(UUID fromId, String fromType, List relationTypeGroups); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java index 1c8603e769..eeff3f8c79 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/JpaRuleNodeDao.java @@ -5,7 +5,7 @@ * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.id.RuleNodeId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -32,6 +33,7 @@ import org.thingsboard.server.dao.sql.JpaAbstractSearchTextDao; import java.util.List; import java.util.Objects; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Component @@ -64,6 +66,11 @@ public class JpaRuleNodeDao extends JpaAbstractSearchTextDao ruleNodeIds) { + ruleNodeRepository.deleteAllById(ruleNodeIds.stream().map(RuleNodeId::getId).collect(Collectors.toList())); + } + @Override public EntityType getEntityType() { return EntityType.RULE_NODE; diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java index f584ff2b02..9fee64824b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rule/RuleNodeRepository.java @@ -39,4 +39,6 @@ public interface RuleNodeRepository extends JpaRepository @Param("searchText") String searchText, Pageable pageable); + void deleteByIdIn(List ids); + }