tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Export/import of attributes

pull/6615/head
Viacheslav Klimov 4 years ago
parent
89c04a3460
commit
32f2f4ccd0
9 changed files with 192 additions and 35 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 79
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
  2. 97
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  3. 15
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  4. 30
      common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java
  5. 2
      common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java
  6. 1
      common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java
  7. 1
      common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java
  8. 1
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java
  9. 1
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java

79
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
View File

@ -19,22 +19,31 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
@Service
@TbCoreComponent
@ -45,6 +54,8 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
private ExportableEntitiesService exportableEntitiesService;
@Autowired
private RelationService relationService;
@Autowired
private AttributesService attributesService;
@Override
public final D getExportData(SecurityUser user, I entityId, EntityExportSettings exportSettings) throws ThingsboardException {
@ -65,22 +76,62 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
protected void setAdditionalExportData(SecurityUser user, E entity, D exportData, EntityExportSettings exportSettings) throws ThingsboardException {
if (exportSettings.isExportRelations()) {
List<EntityRelation> relations = new ArrayList<>();
List<EntityRelation> relations = exportRelations(user, entity);
exportData.setRelations(relations);
}
if (exportSettings.isExportAttributes()) {
Map<String, List<AttributeExportData>> attributes = exportAttributes(user, entity);
exportData.setAttributes(attributes);
}
}
List<EntityRelation> inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
for (EntityRelation relation : inboundRelations) {
exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ);
}
relations.addAll(inboundRelations);
private List<EntityRelation> exportRelations(SecurityUser user, E entity) throws ThingsboardException {
List<EntityRelation> relations = new ArrayList<>();
List<EntityRelation> outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
for (EntityRelation relation : outboundRelations) {
exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ);
}
relations.addAll(outboundRelations);
List<EntityRelation> inboundRelations = relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
for (EntityRelation relation : inboundRelations) {
exportableEntitiesService.checkPermission(user, relation.getFrom(), Operation.READ);
}
relations.addAll(inboundRelations);
exportData.setRelations(relations);
List<EntityRelation> outboundRelations = relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
for (EntityRelation relation : outboundRelations) {
exportableEntitiesService.checkPermission(user, relation.getTo(), Operation.READ);
}
relations.addAll(outboundRelations);
return relations;
}
private Map<String, List<AttributeExportData>> exportAttributes(SecurityUser user, E entity) throws ThingsboardException {
exportableEntitiesService.checkPermission(user, entity, entity.getId().getEntityType(), Operation.READ_ATTRIBUTES);
List<String> scopes;
if (entity.getId().getEntityType() == EntityType.DEVICE) {
scopes = List.of(DataConstants.SERVER_SCOPE, DataConstants.SHARED_SCOPE);
} else {
scopes = Collections.singletonList(DataConstants.SERVER_SCOPE);
}
Map<String, List<AttributeExportData>> attributes = new HashMap<>();
scopes.forEach(scope -> {
try {
attributes.put(scope, attributesService.findAll(user.getTenantId(), entity.getId(), scope).get().stream()
.map(attribute -> {
AttributeExportData attributeExportData = new AttributeExportData();
attributeExportData.setKey(attribute.getKey());
attributeExportData.setLastUpdateTs(attribute.getLastUpdateTs());
attributeExportData.setStrValue(attribute.getStrValue().orElse(null));
attributeExportData.setDoubleValue(attribute.getDoubleValue().orElse(null));
attributeExportData.setLongValue(attribute.getLongValue().orElse(null));
attributeExportData.setBooleanValue(attribute.getBooleanValue().orElse(null));
attributeExportData.setJsonValue(attribute.getJsonValue().orElse(null));
return attributeExportData;
})
.collect(Collectors.toList()));
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
});
return attributes;
}
protected D newExportData() {

97
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
View File

@ -15,7 +15,10 @@
*/
package org.thingsboard.server.service.sync.ie.importing.impl;
import com.google.common.util.concurrent.FutureCallback;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.transaction.annotation.Transactional;
@ -29,23 +32,36 @@ import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.HasId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry;
import org.thingsboard.server.common.data.kv.BooleanDataEntry;
import org.thingsboard.server.common.data.kv.DoubleDataEntry;
import org.thingsboard.server.common.data.kv.JsonDataEntry;
import org.thingsboard.server.common.data.kv.KvEntry;
import org.thingsboard.server.common.data.kv.LongDataEntry;
import org.thingsboard.server.common.data.kv.StringDataEntry;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.service.sync.ie.importing.EntityImportService;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
@Slf4j
public abstract class BaseEntityImportService<I extends EntityId, E extends ExportableEntity<I>, D extends EntityExportData<E>> implements EntityImportService<I, E, D> {
@Autowired @Lazy
@ -53,6 +69,8 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@Autowired
private RelationService relationService;
@Autowired
private TelemetrySubscriptionService tsSubService;
@Autowired
protected EntityActionService entityActionService;
@Autowired
protected TbClusterService clusterService;
@ -102,37 +120,41 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
onEntitySaved(user, savedEntity, oldEntity);
});
importResult.addSaveReferencesCallback(() -> {
if (!importSettings.isUpdateRelations() || exportData.getRelations() == null) {
return;
}
List<EntityRelation> relations = new ArrayList<>(exportData.getRelations());
if (importSettings.isUpdateRelations() && exportData.getRelations() != null) {
importRelations(user, exportData.getRelations(), importResult);
}
if (importSettings.isSaveAttributes() && exportData.getAttributes() != null) {
importAttributes(user, exportData.getAttributes(), importResult);
}
}
private void importRelations(SecurityUser user, List<EntityRelation> relations, EntityImportResult<E> importResult) {
E entity = importResult.getSavedEntity();
importResult.addSaveReferencesCallback(() -> {
for (EntityRelation relation : relations) {
if (!relation.getTo().equals(savedEntity.getId())) {
if (!relation.getTo().equals(entity.getId())) {
HasId<EntityId> to = findInternalEntity(user.getTenantId(), relation.getTo());
exportableEntitiesService.checkPermission(user, to, to.getId().getEntityType(), Operation.WRITE);
relation.setTo(to.getId());
}
if (!relation.getFrom().equals(savedEntity.getId())) {
if (!relation.getFrom().equals(entity.getId())) {
HasId<EntityId> from = findInternalEntity(user.getTenantId(), relation.getFrom());
exportableEntitiesService.checkPermission(user, from, from.getId().getEntityType(), Operation.WRITE);
relation.setFrom(from.getId());
}
}
if (oldEntity != null) {
if (importResult.getOldEntity() != null) {
List<EntityRelation> existingRelations = new ArrayList<>();
existingRelations.addAll(relationService.findByTo(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByFrom(user.getTenantId(), savedEntity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByTo(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByFrom(user.getTenantId(), entity.getId(), RelationTypeGroup.COMMON));
for (EntityRelation existingRelation : existingRelations) {
if (!relations.contains(existingRelation)) {
EntityId otherEntity = null;
if (!existingRelation.getTo().equals(savedEntity.getId())) {
if (!existingRelation.getTo().equals(entity.getId())) {
otherEntity = existingRelation.getTo();
} else if (!existingRelation.getFrom().equals(savedEntity.getId())) {
} else if (!existingRelation.getFrom().equals(entity.getId())) {
otherEntity = existingRelation.getFrom();
}
if (otherEntity != null) {
@ -161,6 +183,44 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
});
}
private void importAttributes(SecurityUser user, Map<String, List<AttributeExportData>> attributes, EntityImportResult<E> importResult) {
E entity = importResult.getSavedEntity();
importResult.addSaveReferencesCallback(() -> {
attributes.forEach((scope, attributesExportData) -> {
List<AttributeKvEntry> attributeKvEntries = attributesExportData.stream()
.map(attributeExportData -> {
KvEntry kvEntry;
String key = attributeExportData.getKey();
if (attributeExportData.getStrValue() != null) {
kvEntry = new StringDataEntry(key, attributeExportData.getStrValue());
} else if (attributeExportData.getBooleanValue() != null) {
kvEntry = new BooleanDataEntry(key, attributeExportData.getBooleanValue());
} else if (attributeExportData.getDoubleValue() != null) {
kvEntry = new DoubleDataEntry(key, attributeExportData.getDoubleValue());
} else if (attributeExportData.getLongValue() != null) {
kvEntry = new LongDataEntry(key, attributeExportData.getLongValue());
} else if (attributeExportData.getJsonValue() != null) {
kvEntry = new JsonDataEntry(key, attributeExportData.getJsonValue());
} else {
throw new IllegalArgumentException("Invalid attribute export data");
}
return new BaseAttributeKvEntry(kvEntry, attributeExportData.getLastUpdateTs());
})
.collect(Collectors.toList());
// fixme: attributes are saved outside the transaction
tsSubService.saveAndNotify(user.getTenantId(), entity.getId(), scope, attributeKvEntries, new FutureCallback<Void>() {
@Override
public void onSuccess(@Nullable Void unused) {}
@Override
public void onFailure(Throwable thr) {
log.error("Failed to import attributes for {} {}", entity.getId().getEntityType(), entity.getId(), thr);
}
});
});
});
}
protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException {
entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity,
savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(),
@ -168,6 +228,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
}
@SuppressWarnings("unchecked")
protected E findExistingEntity(TenantId tenantId, E entity, EntityImportSettings importSettings) {
return (E) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, entity.getId()))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, entity.getId())))
@ -181,6 +242,7 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
.orElse(null);
}
@SuppressWarnings("unchecked")
private <ID extends EntityId> HasId<ID> findInternalEntity(TenantId tenantId, ID externalId) {
return (HasId<ID>) Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndExternalId(tenantId, externalId))
.or(() -> Optional.ofNullable(exportableEntitiesService.findEntityByTenantIdAndId(tenantId, externalId)))
@ -216,7 +278,8 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
EntityId internalId = null;
try {
internalId = getInternalId(externalId);
} catch (Exception ignored) {}
} catch (Exception ignored) {
}
if (internalId != null) {
return Optional.of(internalId);

15
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
View File

@ -27,10 +27,10 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardErrorCode;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
@ -62,8 +62,8 @@ import org.thingsboard.server.common.data.sync.vc.request.load.SingleEntityVersi
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadConfig;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
@ -73,8 +73,6 @@ import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalUnit;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@ -98,6 +96,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private final GitVersionControlQueueService gitServiceQueue;
private final EntitiesExportImportService exportImportService;
private final ExportableEntitiesService exportableEntitiesService;
private final TbNotificationEntityService entityNotificationService;
private final TransactionTemplate transactionTemplate;
private ListeningExecutorService executor;
@ -166,6 +165,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private ListenableFuture<Void> saveEntityData(SecurityUser user, CommitGitRequest commit, EntityId entityId, VersionCreateConfig config) throws Exception {
EntityExportData<ExportableEntity<EntityId>> entityData = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder()
.exportRelations(config.isSaveRelations())
.exportAttributes(config.isSaveAttributes())
.build());
return gitServiceQueue.addToCommit(commit, entityData);
}
@ -208,6 +208,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
try {
return exportImportService.importEntity(user, entityData, EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.findExistingByName(false)
.build(), true, true);
} catch (Exception e) {
@ -246,6 +247,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
for (EntityExportData entityData : entityDataList) {
EntityImportResult<?> importResult = exportImportService.importEntity(user, entityData, EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.findExistingByName(config.isFindExistingEntityByName())
.build(), false, false);
@ -283,6 +285,10 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
exportableEntitiesService.deleteByTenantIdAndId(user.getTenantId(), entity.getId());
sendEventsCallbacks.add(() -> {
entityNotificationService.notifyDeleteEntity(user.getTenantId(), entity.getId(),
entity, null, ActionType.DELETED, null, user);
});
VersionLoadResult result = results.get(entityType);
result.setDeleted(result.getDeleted() + 1);
}
@ -321,6 +327,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
EntityExportData<?> currentVersion = exportImportService.exportEntity(user, entityId, EntityExportSettings.builder()
.exportRelations(true)
.exportAttributes(true)
.build());
return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId),
otherVersion -> transform(gitServiceQueue.getContentsDiff(user.getTenantId(),

30
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/AttributeExportData.java
View File

@ -0,0 +1,30 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.sync.ie;
import lombok.Data;
@Data
public class AttributeExportData {
private String key;
private Long lastUpdateTs;
private Boolean booleanValue;
private String strValue;
private Long longValue;
private Double doubleValue;
private String jsonValue;
}

2
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java
View File

@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.sync.JsonTbEntity;
import java.util.List;
import java.util.Map;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "entityType", include = As.EXISTING_PROPERTY, visible = true, defaultImpl = EntityExportData.class)
@ -45,5 +46,6 @@ public class EntityExportData<E extends ExportableEntity<? extends EntityId>> {
private EntityType entityType;
private List<EntityRelation> relations;
private Map<String, List<AttributeExportData>> attributes;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportSettings.java
View File

@ -26,4 +26,5 @@ import lombok.NoArgsConstructor;
@Builder
public class EntityExportSettings {
private boolean exportRelations;
private boolean exportAttributes;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityImportSettings.java
View File

@ -27,4 +27,5 @@ import lombok.NoArgsConstructor;
public class EntityImportSettings {
private boolean findExistingByName;
private boolean updateRelations;
private boolean saveAttributes;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/create/VersionCreateConfig.java
View File

@ -24,4 +24,5 @@ public class VersionCreateConfig implements Serializable {
private static final long serialVersionUID = 1223723167716612772L;
private boolean saveRelations;
private boolean saveAttributes;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/request/load/VersionLoadConfig.java
View File

@ -21,5 +21,6 @@ import lombok.Data;
public class VersionLoadConfig {
private boolean loadRelations;
private boolean loadAttributes;
}

Write Preview
Loading…
Cancel
Save