tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Resources migration improvements

pull/11873/head
ViacheslavKlimov 2 years ago
parent
1819111145
commit
f621feac53
6 changed files with 104 additions and 69 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
  2. 102
      application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java
  3. 24
      application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java
  4. 3
      common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java
  5. 2
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
  6. 38
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java

4
application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java
View File

@ -510,6 +510,10 @@ public class InstallScripts {
public void updateResourcesUsage() {
resourcesUpdater.updateDashboardsResources();
resourcesUpdater.updateWidgetsResources();
/* TODO TMP remove */
// loadSystemResources(Path.of("/home/viacheslav/Desktop/thingsboard-ce/application/src/main/data/resources/js_modules"), ResourceType.JS_MODULE);
// resourcesUpdater.updateFiles();
}
private void loadSystemResources(Path dir, ResourceType resourceType) {

102
application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java
View File

@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Iterators;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@ -26,8 +27,6 @@ import org.springframework.stereotype.Component;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.HasImage;
import org.thingsboard.server.common.data.ResourceExportData;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.id.EntityId;
@ -142,52 +141,71 @@ public class ResourcesUpdater {
log.info("Updated {} widgets", updatedCount);
}
// TODO: remove after updating PE
@SneakyThrows
public void updateFiles() {
Path widgetsDirectory = Path.of("/home/viacheslav/Desktop/thingsboard-ce/application/src/main/data/json/system/widget_types");
try {
Files.list(widgetsDirectory).forEach(path -> {
WidgetTypeDetails widgetTypeDetails = JacksonUtil.readValue(path.toFile(), WidgetTypeDetails.class);
widgetTypeDetails.setTenantId(TenantId.SYS_TENANT_ID);
Path dashboardsDirectory = Path.of("/home/viacheslav/Desktop/thingsboard-ce/application/src/main/data/json/demo/dashboards");
imageService.updateImagesUsage(widgetTypeDetails);
resourceService.updateResourcesUsage(widgetTypeDetails);
Files.list(widgetsDirectory).forEach(path -> {
WidgetTypeDetails widgetTypeDetails = JacksonUtil.readValue(path.toFile(), WidgetTypeDetails.class);
widgetTypeDetails.setTenantId(TenantId.SYS_TENANT_ID);
Map<TbResourceId, TbResourceInfo> resources = new HashMap<>();
for (TbResourceInfo imageInfo : imageService.getUsedImages(widgetTypeDetails)) {
resources.putIfAbsent(imageInfo.getId(), imageInfo);
}
for (TbResourceInfo resourceInfo : resourceService.getUsedResources(widgetTypeDetails)) {
resources.putIfAbsent(resourceInfo.getId(), resourceInfo);
}
imageService.updateImagesUsage(widgetTypeDetails);
resourceService.updateResourcesUsage(widgetTypeDetails);
widgetTypeDetails.setResources(resources.values().stream()
.map(resourceInfo -> {
if (resourceInfo.getResourceType() == ResourceType.IMAGE) {
ResourceExportData imageExportData = imageService.exportImage(resourceInfo);
imageExportData.setResourceKey(null); // so that the image is not updated by resource key on import
return imageExportData;
} else {
return resourceService.exportResource(resourceInfo);
}
})
.toList());
Map<TbResourceId, TbResourceInfo> resources = new HashMap<>();
for (TbResourceInfo imageInfo : imageService.getUsedImages(widgetTypeDetails)) {
resources.putIfAbsent(imageInfo.getId(), imageInfo);
}
for (TbResourceInfo resourceInfo : resourceService.getUsedResources(widgetTypeDetails)) {
resources.putIfAbsent(resourceInfo.getId(), resourceInfo);
}
ObjectNode json = (ObjectNode) JacksonUtil.valueToTree(widgetTypeDetails);
Iterators.removeIf(json.fields(), field -> field.getValue() == null
|| field.getValue().isNull());
json.remove(List.of("id", "createdTime", "tenantId", "externalId", "version", "scada"));
JsonNode resourcesNode = json.remove("resources");
json.set("resources", resourcesNode);
try {
Files.writeString(path, json.toPrettyString());
log.info("UPDATED {}: {} resources used", path, resources.size());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
} catch (IOException e) {
throw new RuntimeException(e);
}
widgetTypeDetails.setResources(resourceService.exportResources(TenantId.SYS_TENANT_ID, resources.values()));
ObjectNode json = (ObjectNode) JacksonUtil.valueToTree(widgetTypeDetails);
Iterators.removeIf(json.fields(), field -> field.getValue() == null
|| field.getValue().isNull());
json.remove(List.of("id", "createdTime", "tenantId", "externalId", "version", "scada"));
JsonNode resourcesNode = json.remove("resources");
json.set("resources", resourcesNode);
try {
Files.writeString(path, json.toPrettyString());
// log.info("UPDATED {}: {} resources used", path, resources.size());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
Files.list(dashboardsDirectory).forEach(path -> {
Dashboard dashboard = JacksonUtil.readValue(path.toFile(), Dashboard.class);
dashboard.setTenantId(TenantId.SYS_TENANT_ID);
imageService.updateImagesUsage(dashboard);
resourceService.updateResourcesUsage(dashboard);
Map<TbResourceId, TbResourceInfo> resources = new HashMap<>();
for (TbResourceInfo imageInfo : imageService.getUsedImages(dashboard)) {
resources.putIfAbsent(imageInfo.getId(), imageInfo);
}
for (TbResourceInfo resourceInfo : resourceService.getUsedResources(dashboard)) {
resources.putIfAbsent(resourceInfo.getId(), resourceInfo);
}
dashboard.setResources(resourceService.exportResources(TenantId.SYS_TENANT_ID, resources.values()));
ObjectNode json = (ObjectNode) JacksonUtil.valueToTree(dashboard);
json.remove(List.of("id", "createdTime", "tenantId", "externalId", "version", "scada"));
JsonNode resourcesNode = json.remove("resources");
json.set("resources", resourcesNode);
try {
Files.writeString(path, json.toPrettyString());
// log.info("UPDATED {}: {} resources used", path, resources.size());
} catch (IOException e) {
throw new RuntimeException(e);
}
});
}
public void updateDeviceProfilesImages() {

24
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java
View File

@ -164,22 +164,11 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
List<TbResourceInfo> resources = new ArrayList<>();
resources.addAll(imagesProcessor.apply(entity));
resources.addAll(resourcesProcessor.apply(entity));
for (TbResourceInfo resourceInfo : resources) {
accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resourceInfo.getId(), resourceInfo);
}
return resources.stream()
.map(resourceInfo -> {
if (resourceInfo.getResourceType() == ResourceType.IMAGE) {
ResourceExportData imageExportData = imageService.exportImage(resourceInfo);
imageExportData.setResourceKey(null); // so that the image is not updated by resource key on import
return imageExportData;
} else {
return resourceService.exportResource(resourceInfo);
}
})
.toList();
return resourceService.exportResources(user.getTenantId(), resources);
}
private TbResourceInfo importResource(ResourceExportData resourceData, SecurityUser user) throws ThingsboardException {
@ -187,12 +176,13 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
resource.setTenantId(user.getTenantId());
accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.CREATE, null, resource);
TbResourceInfo resourceInfo = resourceService.toResource(user.getTenantId(), resourceData);
if (resourceInfo.getId() != null) {
accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resourceInfo.getId(), resourceInfo);
return resourceInfo;
resource = resourceService.toResource(user.getTenantId(), resourceData);
if (resource.getData() != null) {
return save(resource, user);
} else {
accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resource.getId(), resource);
return resource;
}
return save(resource, user);
}
private Comparator<? super LwM2mObject> getComparator(String sortProperty, String sortOrder) {

3
common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java
View File

@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.dao.entity.EntityDaoService;
import java.util.Collection;
import java.util.List;
public interface ResourceService extends EntityDaoService {
@ -45,6 +46,8 @@ public interface ResourceService extends EntityDaoService {
ResourceExportData exportResource(TbResourceInfo resourceInfo);
List<ResourceExportData> exportResources(TenantId tenantId, Collection<TbResourceInfo> resources);
TbResource toResource(TenantId tenantId, ResourceExportData exportData);
void importResources(TenantId tenantId, List<ResourceExportData> resources);

2
dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
View File

@ -256,6 +256,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
String etag = calculateImageEtag(data);
TbResourceInfo existingImage = findSystemOrTenantImageByEtag(tenantId, etag);
if (existingImage != null) {
log.info("[{}] Using existing image {}", tenantId, existingImage.getLink());
return new TbResource(existingImage);
}
}
@ -281,6 +282,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
descriptor.setMediaType(imageData.getMediaType());
image.setDescriptorValue(descriptor);
image.setData(data);
log.info("[{}] Creating image {}", tenantId, image.getFileName());
return image;
}

38
dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java
View File

@ -59,6 +59,7 @@ import org.thingsboard.server.dao.service.validator.ResourceDataValidator;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Base64;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@ -203,6 +204,21 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
.build();
}
@Override
public List<ResourceExportData> exportResources(TenantId tenantId, Collection<TbResourceInfo> resources) {
return resources.stream()
.map(resourceInfo -> {
if (resourceInfo.getResourceType() == ResourceType.IMAGE) {
ResourceExportData imageExportData = imageService.exportImage(resourceInfo);
imageExportData.setResourceKey(null); // so that the image is not updated by resource key on import
return imageExportData;
} else {
return exportResource(resourceInfo);
}
})
.toList();
}
@Override
public void importResources(TenantId tenantId, List<ResourceExportData> resources) {
for (ResourceExportData resourceData : resources) {
@ -237,17 +253,18 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
String etag = calculateEtag(data);
TbResourceInfo existingResource;
boolean updateData = false;
boolean update = false;
if (!tenantId.isSysTenantId()) {
existingResource = findSystemOrTenantResourceByEtag(tenantId, exportData.getType(), etag);
} else {
existingResource = findResourceInfoByTenantIdAndKey(tenantId, exportData.getType(), exportData.getResourceKey());
updateData = true; // we overwrite system resource instead of creating new
update = true; // we overwrite system resource instead of creating new
}
if (existingResource != null) {
TbResource resource = new TbResource(existingResource);
if (updateData && !etag.equals(resource.getEtag())) {
if (update && !etag.equals(resource.getEtag())) {
resource.setData(data);
resource.setTitle(exportData.getTitle());
log.info("[{}] Updating existing resource {}", tenantId, existingResource.getLink());
} else {
log.info("[{}] Using existing resource {}", tenantId, existingResource.getLink());
@ -416,15 +433,13 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
}
private boolean updateResourcesUsage(TenantId tenantId, JsonNode jsonNode, Map<String, String> mapping, Map<String, String> links) {
AtomicBoolean updated = new AtomicBoolean(false);
processResources(jsonNode, mapping, value -> {
return processResources(jsonNode, mapping, value -> {
String link = getResourceLink(value);
if (link != null) {
String newLink = links.get(link);
if (newLink == null || newLink.equals(link)) {
return value; // leaving link as is
} else {
updated.set(true);
return DataConstants.TB_RESOURCE_PREFIX + newLink;
}
} else { // probably importing an old dashboard json where resources are referenced by ids
@ -435,7 +450,6 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
return value;
}
TbResourceInfo resourceInfo = findResourceInfoById(tenantId, resourceId);
updated.set(true);
if (resourceInfo != null) {
return DataConstants.TB_RESOURCE_PREFIX + resourceInfo.getLink();
} else {
@ -444,7 +458,6 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
}
}
});
return updated.get();
}
@Override
@ -503,7 +516,8 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
}
}
private void processResources(JsonNode jsonNode, Map<String, String> mapping, UnaryOperator<String> processor) {
private boolean processResources(JsonNode jsonNode, Map<String, String> mapping, UnaryOperator<String> processor) {
AtomicBoolean updated = new AtomicBoolean(false);
JacksonUtil.replaceByMapping(jsonNode, mapping, Collections.emptyMap(), (name, urlNode) -> {
String value = null;
if (urlNode.isTextual()) { // link is in the right place
@ -522,9 +536,13 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
}
JsonNode newValue = new TextNode(value);
log.trace("Replaced '{}' with '{}'", urlNode, newValue);
if (!newValue.toString().equals(urlNode.toString())) {
updated.set(true);
log.info("Replaced '{}' with '{}'", urlNode, newValue);
}
return newValue;
});
return updated.get();
}
@Override

Write Preview
Loading…
Cancel
Save