From ab5d6f3c0dcb6ad301fbd26005854632c104bdc0 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Wed, 30 Mar 2022 13:31:43 +0300 Subject: [PATCH] Entities export/import API refactoring --- .../EntitiesExportImportController.java | 144 ++++++++++++++---- .../DefaultEntitiesExportImportService.java | 26 ++++ .../EntitiesExportImportService.java | 4 + .../exporting/data/EntityExportData.java | 2 + .../server/common/data/Customer.java | 35 +---- .../server/common/data/Dashboard.java | 27 +--- .../server/dao/ExportableEntityDao.java | 7 +- 7 files changed, 153 insertions(+), 92 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java index fc3b0b712b..ec2d4273cc 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesExportImportController.java @@ -27,23 +27,37 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityFilter; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.exportimport.EntitiesExportImportService; import org.thingsboard.server.service.exportimport.exporting.EntityExportSettings; import org.thingsboard.server.service.exportimport.exporting.data.EntityExportData; import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import java.util.Collections; +import java.util.List; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.dao.sql.query.EntityKeyMapping.CREATED_TIME; + @RestController @RequestMapping("/api/entities") @TbCoreComponent @@ -51,7 +65,58 @@ import java.util.stream.Collectors; public class EntitiesExportImportController extends BaseController { private final EntitiesExportImportService exportImportService; + private final EntityQueryService entityQueryService; + + + // TODO [viacheslav]: export and import of batches + // TODO [viacheslav]: api to export and import whole customer, whole tenant + + + @PostMapping("/exportByFilter") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List> exportEntitiesByFilter(@RequestBody EntityFilter filter, + @RequestParam(defaultValue = "false") boolean exportInboundRelations, + @RequestParam(defaultValue = "0") int page, + @RequestParam(defaultValue = "100") int pageSize) throws ThingsboardException { + EntityDataPageLink pageLink = new EntityDataPageLink(); + pageLink.setPage(page); + pageLink.setPageSize(pageSize); + pageLink.setSortOrder(new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME), EntityDataSortOrder.Direction.DESC)); + EntityDataQuery entityDataQuery = new EntityDataQuery(filter, pageLink, List.of(new EntityKey(EntityKeyType.ENTITY_FIELD, CREATED_TIME)), + Collections.emptyList(), Collections.emptyList()); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { + // FIXME [viacheslav]: check read permission for relation fromId + return entityQueryService.findEntityDataByQuery(user, entityDataQuery).getData().stream() + .map(EntityData::getEntityId) + .map(entityId -> { + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + }) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); + } + } + @PostMapping("/exportByQuery") + @PreAuthorize("hasAuthority('TENANT_ADMIN')") + public List> exportEntitiesByQuery(@RequestBody EntityDataQuery query, + @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); +// FIXME [viacheslav]: check read permission for relation fromId + try { + return entityQueryService.findEntityDataByQuery(user, query).getData().stream() + .map(EntityData::getEntityId) + .map(entityId -> exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings)) + .collect(Collectors.toList()); + } catch (Exception e) { + throw handleException(e); + } + } @PostMapping("/export/{entityType}/{entityId}") @PreAuthorize("hasAuthority('TENANT_ADMIN')") @@ -60,11 +125,13 @@ public class EntitiesExportImportController extends BaseController { @PathVariable("entityId") UUID entityUuid, @RequestParam(defaultValue = "false") boolean exportInboundRelations) throws ThingsboardException { EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, entityUuid); - EntityExportSettings exportSettings = EntityExportSettings.builder() - .exportInboundRelations(exportInboundRelations) - .build(); - try { - return exportEntity(getCurrentUser(), entityId, exportSettings); + checkEntityId(entityId, Operation.READ); + + SecurityUser user = getCurrentUser(); + EntityExportSettings exportSettings = toExportSettings(exportInboundRelations); + + try { // FIXME [viacheslav]: check read permission for relation fromId + return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); } catch (Exception e) { throw handleException(e); } @@ -73,45 +140,64 @@ public class EntitiesExportImportController extends BaseController { @PostMapping("/import") @PreAuthorize("hasAuthority('TENANT_ADMIN')") - public EntityImportResult> importEntity(@RequestBody EntityExportData> exportData, - @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { - EntityImportSettings importSettings = EntityImportSettings.builder() - .importInboundRelations(importInboundRelations) - .build(); + public List>> importEntity(@RequestBody List>> exportDataList, + @RequestParam(defaultValue = "false") boolean importInboundRelations) throws ThingsboardException { + SecurityUser user = getCurrentUser(); + EntityImportSettings importSettings = toImportSettings(importInboundRelations); + + for (EntityExportData> exportData : exportDataList) { + checkPermissionsForImport(user, exportData, importSettings); + } + try { - return importEntity(getCurrentUser(), exportData, importSettings); + return exportImportService.importEntities(user.getTenantId(), exportDataList, importSettings); } catch (Exception e) { throw handleException(e); } } - - // TODO [viacheslav]: export and import of batches - // TODO [viacheslav]: api to export and import whole customer, whole tenant - - private EntityExportData> exportEntity(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { + public void checkPermissionsForExport(SecurityUser user, EntityId entityId, EntityExportSettings exportSettings) throws ThingsboardException { checkEntityId(entityId, Operation.READ); - return exportImportService.exportEntity(user.getTenantId(), entityId, exportSettings); + if (exportSettings.isExportInboundRelations()) { + for (EntityRelation entityRelation : relationService.findByTo(user.getTenantId(), entityId, RelationTypeGroup.COMMON)) { + EntityId fromId = entityRelation.getFrom(); + checkEntityId(fromId, Operation.READ); + } + } } - private EntityImportResult> importEntity(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { - ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); - if (existingEntity != null) {// todo [viacheslav] maybe need to extract permission check to BaseController and put there permission checks from other controllers + public void checkPermissionsForImport(SecurityUser user, EntityExportData> exportData, EntityImportSettings importSettings) throws ThingsboardException { + ExportableEntity existingEntity = exportImportService.findEntityByExternalId(user.getTenantId(), exportData.getMainEntity().getId()); + if (existingEntity != null) { checkEntityId(existingEntity.getId(), Operation.WRITE); - if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { - for (EntityId fromId : exportData.getInboundRelations().stream().map(EntityRelation::getFrom).collect(Collectors.toSet())) { - // FIXME [viacheslav]: fromId is external -// checkEntityId(fromId, Operation.WRITE); + } else { + accessControlService.checkPermission(user, Resource.of(exportData.getEntityType()), Operation.CREATE); + } + + if (importSettings.isImportInboundRelations() && CollectionUtils.isNotEmpty(exportData.getInboundRelations())) { + for (EntityRelation entityRelation : exportData.getInboundRelations()) { + ExportableEntity entityFrom = exportImportService.findEntityByExternalId(user.getTenantId(), entityRelation.getFrom()); + if (entityFrom != null) { + accessControlService.checkPermission(user, Resource.of(entityFrom.getId().getEntityType()), Operation.WRITE, entityFrom.getId(), entityFrom); + } else { + throw new ThingsboardException("Relation with non-existing entity", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } } - } else { - checkEntity(null, exportData.getMainEntity(), Resource.of(exportData.getEntityType())); } + } + - EntityImportResult> importResult = exportImportService.importEntity(getTenantId(), exportData, importSettings); - onEntityUpdatedOrCreated(user, importResult.getSavedEntity(), importResult.getOldEntity(), importResult.getOldEntity() == null); + private EntityImportSettings toImportSettings(boolean importInboundRelations) { + EntityImportSettings importSettings = EntityImportSettings.builder() + .importInboundRelations(importInboundRelations) + .build(); + return importSettings; + } - return importResult; + private EntityExportSettings toExportSettings(boolean exportInboundRelations) { + return EntityExportSettings.builder() + .exportInboundRelations(exportInboundRelations) + .build(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java index 4e9c42cffd..93fa370c94 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/DefaultEntitiesExportImportService.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.exportimport; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.id.EntityId; @@ -31,9 +32,12 @@ import org.thingsboard.server.service.exportimport.importing.EntityImportService import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; import java.util.Collection; +import java.util.Comparator; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; // FIXME [viacheslav]: review packages and classes naming @Service @@ -44,6 +48,11 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> importServices = new HashMap<>(); private final Map> daos = new HashMap<>(); + protected static final List SUPPORTED_ENTITY_TYPES = List.of( + EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, + EntityType.DEVICE_PROFILE, EntityType.DEVICE, EntityType.DASHBOARD + ); + // TODO [viacheslav]: export and import of the whole tenant // TODO [viacheslav]: export and import of the whole customer ? @@ -55,7 +64,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return exportService.getExportData(tenantId, entityId, exportSettings); } + // FIXME [viacheslav]: somehow validate export data + @Transactional @Override public , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings) { EntityType entityType = exportData.getEntityType(); @@ -64,6 +75,15 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS return importService.importEntity(tenantId, exportData, importSettings); } + @Transactional + @Override + public , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings) { + return exportDataList.stream() + .sorted(Comparator.comparing(exportData -> SUPPORTED_ENTITY_TYPES.indexOf(exportData.getEntityType()))) + .map(exportData -> importEntity(tenantId, exportData, importSettings)) + .collect(Collectors.toList()); + } + @Override public , I extends EntityId> E findEntityById(TenantId tenantId, I id) { @@ -81,11 +101,17 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityExportService getExportService(EntityType entityType) { + if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + throw new IllegalArgumentException("Export for entity type " + entityType + " is not supported"); + } return (EntityExportService) exportServices.get(entityType); } @SuppressWarnings("unchecked") private , D extends EntityExportData> EntityImportService getImportService(EntityType entityType) { + if (!SUPPORTED_ENTITY_TYPES.contains(entityType)) { + throw new IllegalArgumentException("Import for entity type " + entityType + " is not supported"); + } return (EntityImportService) importServices.get(entityType); } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java index 1eb085d219..e8d4a40efe 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/EntitiesExportImportService.java @@ -23,10 +23,14 @@ import org.thingsboard.server.service.exportimport.exporting.data.EntityExportDa import org.thingsboard.server.service.exportimport.importing.EntityImportResult; import org.thingsboard.server.service.exportimport.importing.EntityImportSettings; +import java.util.List; + public interface EntitiesExportImportService extends ExportableEntitiesService { , I extends EntityId> EntityExportData exportEntity(TenantId tenantId, I entityId, EntityExportSettings exportSettings); , I extends EntityId> EntityImportResult importEntity(TenantId tenantId, EntityExportData exportData, EntityImportSettings importSettings); + , I extends EntityId> List> importEntities(TenantId tenantId, List> exportDataList, EntityImportSettings importSettings); + } diff --git a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java index 74116276cb..f77a5fbb29 100644 --- a/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java +++ b/application/src/main/java/org/thingsboard/server/service/exportimport/exporting/data/EntityExportData.java @@ -17,6 +17,7 @@ package org.thingsboard.server.service.exportimport.exporting.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -42,6 +43,7 @@ import java.util.List; public abstract class EntityExportData> { private E mainEntity; + @JsonInclude(JsonInclude.Include.NON_NULL) private List inboundRelations; @JsonIgnore diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java index b7287c55af..92e5e9ae78 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Customer.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.CustomerId; @@ -27,6 +28,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; +@EqualsAndHashCode(callSuper = false) public class Customer extends ContactBased implements HasTenantId, ExportableEntity { private static final long serialVersionUID = -1599722990298929275L; @@ -39,7 +41,7 @@ public class Customer extends ContactBased implements HasTenantId, E private TenantId tenantId; @Getter @Setter - private CustomerId externalId; // FIXME [viacheslav]: add to hashcode, equals, etc + private CustomerId externalId; public Customer() { super(); @@ -167,37 +169,6 @@ public class Customer extends ContactBased implements HasTenantId, E return getTitle(); } - @Override - public int hashCode() { - final int prime = 31; - int result = super.hashCode(); - result = prime * result + ((tenantId == null) ? 0 : tenantId.hashCode()); - result = prime * result + ((title == null) ? 0 : title.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (!super.equals(obj)) - return false; - if (getClass() != obj.getClass()) - return false; - Customer other = (Customer) obj; - if (tenantId == null) { - if (other.tenantId != null) - return false; - } else if (!tenantId.equals(other.tenantId)) - return false; - if (title == null) { - if (other.title != null) - return false; - } else if (!title.equals(other.title)) - return false; - return true; - } - @Override public String toString() { StringBuilder builder = new StringBuilder(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index dce910c4ff..59c122f3d0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -17,10 +17,12 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.databind.JsonNode; import io.swagger.annotations.ApiModelProperty; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; +@EqualsAndHashCode(callSuper = false) public class Dashboard extends DashboardInfo implements ExportableEntity { private static final long serialVersionUID = 872682138346187503L; @@ -60,31 +62,6 @@ public class Dashboard extends DashboardInfo implements ExportableEntity { T findByTenantIdAndExternalId(UUID tenantId, UUID externalId); T findByTenantIdAndId(UUID tenantId, UUID id); - // fixme [viacheslav]: get rid of boilerplate ? - EntityType getEntityType(); - /* - * default > ExportableEntityRepository getExportableEntityRepository() { - * ((ExportableEntityRepository) getJpaRepository).find... - * */ + EntityType getEntityType(); }