From 32f2f4ccd07c044b9396c42b556146c4daa3c832 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 31 May 2022 13:19:36 +0300 Subject: [PATCH] Export/import of attributes --- .../impl/DefaultEntityExportService.java | 79 ++++++++++++--- .../impl/BaseEntityImportService.java | 97 +++++++++++++++---- .../DefaultEntitiesVersionControlService.java | 15 ++- .../data/sync/ie/AttributeExportData.java | 30 ++++++ .../common/data/sync/ie/EntityExportData.java | 2 + .../data/sync/ie/EntityExportSettings.java | 1 + .../data/sync/ie/EntityImportSettings.java | 1 + .../request/create/VersionCreateConfig.java | 1 + .../vc/request/load/VersionLoadConfig.java | 1 + 9 files changed, 192 insertions(+), 35 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 31bf768e90..8e723f8793 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -19,22 +19,31 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.DataConstants; +import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ExportableEntity; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; +import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityExportSettings; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; @Service @TbCoreComponent @@ -45,6 +54,8 @@ public class DefaultEntityExportService relations = new ArrayList<>(); + List relations = exportRelations(user, entity); + exportData.setRelations(relations); + } + if (exportSettings.isExportAttributes()) { + Map> attributes = exportAttributes(user, entity); + exportData.setAttributes(attributes); + } + } - List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : inboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); - } - relations.addAll(inboundRelations); + private List exportRelations(SecurityUser user, E entity) throws ThingsboardException { + List relations = new ArrayList<>(); - List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); - for (EntityRelation relation : outboundRelations) { - exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); - } - relations.addAll(outboundRelations); + List inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : inboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ); + } + relations.addAll(inboundRelations); - exportData.setRelations(relations); + List outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + for (EntityRelation relation : outboundRelations) { + exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ); } + relations.addAll(outboundRelations); + return relations; + } + + private Map> exportAttributes(SecurityUser user, E entity) throws ThingsboardException { + exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ_ATTRIBUTES); + + List scopes; + if (entity.getId().getEntityType() == EntityType.DEVICE) { + scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE); + } else { + scopes = Collections.singletonList(DataConstants.SERVER_SCOPE); + } + Map> attributes = new HashMap<>(); + scopes.forEach(scope -> { + try { + attributes.put(scope, attributesService.findAll(user.getTenantId(), entity.getId(), scope).get().stream() + .map(attribute -> { + AttributeExportData attributeExportData = new AttributeExportData(); + attributeExportData.setKey(attribute.getKey()); + attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs()); + attributeExportData.setStrValue(attribute.getStrValue().orElse(null)); + attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null)); + attributeExportData.setLongValue(attribute.getLongValue().orElse(null)); + attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null)); + attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null)); + return attributeExportData; + }) + .collect(Collectors.toList())); + } catch (InterruptedException | ExecutionException e) { + throw new RuntimeException(e); + } + }); + return attributes; } protected D newExportData() { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java index 203b0caa16..caa1e20ad7 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.service.sync.ie.importing.impl; +import com.google.common.util.concurrent.FutureCallback; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.transaction.annotation.Transactional; @@ -29,23 +32,36 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.EntityIdFactory; import org.thingsboard.server.common.data.id.HasId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.kv.AttributeKvEntry; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.sync.ie.AttributeExportData; +import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.sync.ie.EntityImportResult; +import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.permission.Operation; -import org.thingsboard.server.common.data.sync.ie.EntityImportResult; -import org.thingsboard.server.common.data.sync.ie.EntityImportSettings; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.stream.Collectors; +@Slf4j public abstract class BaseEntityImportService, D extends EntityExportData> implements EntityImportService { @Autowired @Lazy @@ -53,6 +69,8 @@ public abstract class BaseEntityImportService { - if (!importSettings.isUpdateRelations() || exportData.getRelations() == null) { - return; - } - - List relations = new ArrayList<>(exportData.getRelations()); + if (importSettings.isUpdateRelations() && exportData.getRelations() != null) { + importRelations(user, exportData.getRelations(), importResult); + } + if (importSettings.isSaveAttributes() && exportData.getAttributes() != null) { + importAttributes(user, exportData.getAttributes(), importResult); + } + } + private void importRelations(SecurityUser user, List relations, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { for (EntityRelation relation : relations) { - if (!relation.getTo().equals(savedEntity.getId())) { + if (!relation.getTo().equals(entity.getId())) { HasId to = findInternalEntity(user.getTenantId(), relation.getTo()); exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE); relation.setTo(to.getId()); } - if (!relation.getFrom().equals(savedEntity.getId())) { + if (!relation.getFrom().equals(entity.getId())) { HasId from = findInternalEntity(user.getTenantId(), relation.getFrom()); exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE); relation.setFrom(from.getId()); } } - if (oldEntity != null) { + if (importResult.getOldEntity() != null) { List existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON)); for (EntityRelation existingRelation : existingRelations) { if (!relations.contains(existingRelation)) { EntityId otherEntity = null; - if (!existingRelation.getTo().equals(savedEntity.getId())) { + if (!existingRelation.getTo().equals(entity.getId())) { otherEntity = existingRelation.getTo(); - } else if (!existingRelation.getFrom().equals(savedEntity.getId())) { + } else if (!existingRelation.getFrom().equals(entity.getId())) { otherEntity = existingRelation.getFrom(); } if (otherEntity != null) { @@ -161,6 +183,44 @@ public abstract class BaseEntityImportService> attributes, EntityImportResult importResult) { + E entity = importResult.getSavedEntity(); + importResult.addSaveReferencesCallback(() -> { + attributes.forEach((scope, attributesExportData) -> { + List attributeKvEntries = attributesExportData.stream() + .map(attributeExportData -> { + KvEntry kvEntry; + String key = attributeExportData.getKey(); + if (attributeExportData.getStrValue() != null) { + kvEntry = new StringDataEntry(key, attributeExportData.getStrValue()); + } else if (attributeExportData.getBooleanValue() != null) { + kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue()); + } else if (attributeExportData.getDoubleValue() != null) { + kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue()); + } else if (attributeExportData.getLongValue() != null) { + kvEntry = new LongDataEntry(key, attributeExportData.getLongValue()); + } else if (attributeExportData.getJsonValue() != null) { + kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue()); + } else { + throw new IllegalArgumentException("Invalid attribute export data"); + } + return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs()); + }) + .collect(Collectors.toList()); + // fixme: attributes are saved outside the transaction + tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback() { + @Override + public void onSuccess(@Nullable Void unused) {} + + @Override + public void onFailure(Throwable thr) { + log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr); + } + }); + }); + }); + } + protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException { entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity, savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(), @@ -168,6 +228,7 @@ public abstract class BaseEntityImportService Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId()))) @@ -181,6 +242,7 @@ public abstract class BaseEntityImportService HasId findInternalEntity(TenantId tenantId, ID externalId) { return (HasId) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId)) .or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId))) @@ -216,7 +278,8 @@ public abstract class BaseEntityImportService saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception { EntityExportData> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(config.isSaveRelations()) + .exportAttributes(config.isSaveAttributes()) .build()); return gitServiceQueue.addToCommit(commit, entityData); } @@ -208,6 +208,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { return exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(false) .build(), true, true); } catch (Exception e) { @@ -246,6 +247,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont for (EntityExportData entityData : entityDataList) { EntityImportResult importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder() .updateRelations(config.isLoadRelations()) + .saveAttributes(config.isLoadAttributes()) .findExistingByName(config.isFindExistingEntityByName()) .build(), false, false); @@ -283,6 +285,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId()); + sendEventsCallbacks.add(() -> { + entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(), + entity, null, ActionType.DELETED, null, user); + }); VersionLoadResult result = results.get(entityType); result.setDeleted(result.getDeleted() + 1); } @@ -321,6 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityExportData currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder() .exportRelations(true) + .exportAttributes(true) .build()); return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(), diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java new file mode 100644 index 0000000000..dcf69c0877 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java @@ -0,0 +1,30 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.sync.ie; + +import lombok.Data; + +@Data +public class AttributeExportData { + private String key; + private Long lastUpdateTs; + + private Boolean booleanValue; + private String strValue; + private Long longValue; + private Double doubleValue; + private String jsonValue; +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 35fceafe43..542e675940 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.sync.JsonTbEntity; import java.util.List; +import java.util.Map; @JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class) @@ -45,5 +46,6 @@ public class EntityExportData> { private EntityType entityType; private List relations; + private Map> attributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java index 051745c07f..0800a1f7c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java @@ -26,4 +26,5 @@ import lombok.NoArgsConstructor; @Builder public class EntityExportSettings { private boolean exportRelations; + private boolean exportAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java index 3e18bc9f5f..564b0134e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java @@ -27,4 +27,5 @@ import lombok.NoArgsConstructor; public class EntityImportSettings { private boolean findExistingByName; private boolean updateRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java index 10971f1170..4426717fe9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java @@ -24,4 +24,5 @@ public class VersionCreateConfig implements Serializable { private static final long serialVersionUID = 1223723167716612772L; private boolean saveRelations; + private boolean saveAttributes; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java index 8ed9c34694..2d68a2b7e3 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java @@ -21,5 +21,6 @@ import lombok.Data; public class VersionLoadConfig { private boolean loadRelations; + private boolean loadAttributes; }