From f621feac53c5b9ed7540de434dd34bdb3540405e Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Wed, 6 Nov 2024 14:43:31 +0200 Subject: [PATCH] Resources migration improvements --- .../service/install/InstallScripts.java | 4 + .../install/update/ResourcesUpdater.java | 102 ++++++++++-------- .../resource/DefaultTbResourceService.java | 24 ++--- .../server/dao/resource/ResourceService.java | 3 + .../server/dao/resource/BaseImageService.java | 2 + .../dao/resource/BaseResourceService.java | 38 +++++-- 6 files changed, 104 insertions(+), 69 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java index 58cd30b40b..911eee78f6 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java +++ b/application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java @@ -510,6 +510,10 @@ public class InstallScripts { public void updateResourcesUsage() { resourcesUpdater.updateDashboardsResources(); resourcesUpdater.updateWidgetsResources(); + + /* TODO TMP remove */ +// loadSystemResources(Path.of("/home/viacheslav/Desktop/thingsboard-ce/application/src/main/data/resources/js_modules"), ResourceType.JS_MODULE); +// resourcesUpdater.updateFiles(); } private void loadSystemResources(Path dir, ResourceType resourceType) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java index a57e1a29a5..fe9601c0fc 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.Iterators; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -26,8 +27,6 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.HasImage; -import org.thingsboard.server.common.data.ResourceExportData; -import org.thingsboard.server.common.data.ResourceType; import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.EntityId; @@ -142,52 +141,71 @@ public class ResourcesUpdater { log.info("Updated {} widgets", updatedCount); } + // TODO: remove after updating PE + @SneakyThrows public void updateFiles() { Path widgetsDirectory = Path.of("/home/viacheslav/Desktop/thingsboard-ce/application/src/main/data/json/system/widget_types"); - try { - Files.list(widgetsDirectory).forEach(path -> { - WidgetTypeDetails widgetTypeDetails = JacksonUtil.readValue(path.toFile(), WidgetTypeDetails.class); - widgetTypeDetails.setTenantId(TenantId.SYS_TENANT_ID); + Path dashboardsDirectory = Path.of("/home/viacheslav/Desktop/thingsboard-ce/application/src/main/data/json/demo/dashboards"); - imageService.updateImagesUsage(widgetTypeDetails); - resourceService.updateResourcesUsage(widgetTypeDetails); + Files.list(widgetsDirectory).forEach(path -> { + WidgetTypeDetails widgetTypeDetails = JacksonUtil.readValue(path.toFile(), WidgetTypeDetails.class); + widgetTypeDetails.setTenantId(TenantId.SYS_TENANT_ID); - Map resources = new HashMap<>(); - for (TbResourceInfo imageInfo : imageService.getUsedImages(widgetTypeDetails)) { - resources.putIfAbsent(imageInfo.getId(), imageInfo); - } - for (TbResourceInfo resourceInfo : resourceService.getUsedResources(widgetTypeDetails)) { - resources.putIfAbsent(resourceInfo.getId(), resourceInfo); - } + imageService.updateImagesUsage(widgetTypeDetails); + resourceService.updateResourcesUsage(widgetTypeDetails); - widgetTypeDetails.setResources(resources.values().stream() - .map(resourceInfo -> { - if (resourceInfo.getResourceType() == ResourceType.IMAGE) { - ResourceExportData imageExportData = imageService.exportImage(resourceInfo); - imageExportData.setResourceKey(null); // so that the image is not updated by resource key on import - return imageExportData; - } else { - return resourceService.exportResource(resourceInfo); - } - }) - .toList()); + Map resources = new HashMap<>(); + for (TbResourceInfo imageInfo : imageService.getUsedImages(widgetTypeDetails)) { + resources.putIfAbsent(imageInfo.getId(), imageInfo); + } + for (TbResourceInfo resourceInfo : resourceService.getUsedResources(widgetTypeDetails)) { + resources.putIfAbsent(resourceInfo.getId(), resourceInfo); + } - ObjectNode json = (ObjectNode) JacksonUtil.valueToTree(widgetTypeDetails); - Iterators.removeIf(json.fields(), field -> field.getValue() == null - || field.getValue().isNull()); - json.remove(List.of("id", "createdTime", "tenantId", "externalId", "version", "scada")); - JsonNode resourcesNode = json.remove("resources"); - json.set("resources", resourcesNode); - try { - Files.writeString(path, json.toPrettyString()); - log.info("UPDATED {}: {} resources used", path, resources.size()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } catch (IOException e) { - throw new RuntimeException(e); - } + widgetTypeDetails.setResources(resourceService.exportResources(TenantId.SYS_TENANT_ID, resources.values())); + + ObjectNode json = (ObjectNode) JacksonUtil.valueToTree(widgetTypeDetails); + Iterators.removeIf(json.fields(), field -> field.getValue() == null + || field.getValue().isNull()); + json.remove(List.of("id", "createdTime", "tenantId", "externalId", "version", "scada")); + JsonNode resourcesNode = json.remove("resources"); + json.set("resources", resourcesNode); + try { + Files.writeString(path, json.toPrettyString()); +// log.info("UPDATED {}: {} resources used", path, resources.size()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + Files.list(dashboardsDirectory).forEach(path -> { + Dashboard dashboard = JacksonUtil.readValue(path.toFile(), Dashboard.class); + dashboard.setTenantId(TenantId.SYS_TENANT_ID); + + imageService.updateImagesUsage(dashboard); + resourceService.updateResourcesUsage(dashboard); + + Map resources = new HashMap<>(); + for (TbResourceInfo imageInfo : imageService.getUsedImages(dashboard)) { + resources.putIfAbsent(imageInfo.getId(), imageInfo); + } + for (TbResourceInfo resourceInfo : resourceService.getUsedResources(dashboard)) { + resources.putIfAbsent(resourceInfo.getId(), resourceInfo); + } + + dashboard.setResources(resourceService.exportResources(TenantId.SYS_TENANT_ID, resources.values())); + + ObjectNode json = (ObjectNode) JacksonUtil.valueToTree(dashboard); + json.remove(List.of("id", "createdTime", "tenantId", "externalId", "version", "scada")); + JsonNode resourcesNode = json.remove("resources"); + json.set("resources", resourcesNode); + try { + Files.writeString(path, json.toPrettyString()); +// log.info("UPDATED {}: {} resources used", path, resources.size()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); } public void updateDeviceProfilesImages() { diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java index c887ba4a52..9708b46cef 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java @@ -164,22 +164,11 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements List resources = new ArrayList<>(); resources.addAll(imagesProcessor.apply(entity)); resources.addAll(resourcesProcessor.apply(entity)); - for (TbResourceInfo resourceInfo : resources) { accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resourceInfo.getId(), resourceInfo); } - return resources.stream() - .map(resourceInfo -> { - if (resourceInfo.getResourceType() == ResourceType.IMAGE) { - ResourceExportData imageExportData = imageService.exportImage(resourceInfo); - imageExportData.setResourceKey(null); // so that the image is not updated by resource key on import - return imageExportData; - } else { - return resourceService.exportResource(resourceInfo); - } - }) - .toList(); + return resourceService.exportResources(user.getTenantId(), resources); } private TbResourceInfo importResource(ResourceExportData resourceData, SecurityUser user) throws ThingsboardException { @@ -187,12 +176,13 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements resource.setTenantId(user.getTenantId()); accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.CREATE, null, resource); - TbResourceInfo resourceInfo = resourceService.toResource(user.getTenantId(), resourceData); - if (resourceInfo.getId() != null) { - accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resourceInfo.getId(), resourceInfo); - return resourceInfo; + resource = resourceService.toResource(user.getTenantId(), resourceData); + if (resource.getData() != null) { + return save(resource, user); + } else { + accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resource.getId(), resource); + return resource; } - return save(resource, user); } private Comparator getComparator(String sortProperty, String sortOrder) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java index 197bced2f9..529326b14c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.dao.entity.EntityDaoService; +import java.util.Collection; import java.util.List; public interface ResourceService extends EntityDaoService { @@ -45,6 +46,8 @@ public interface ResourceService extends EntityDaoService { ResourceExportData exportResource(TbResourceInfo resourceInfo); + List exportResources(TenantId tenantId, Collection resources); + TbResource toResource(TenantId tenantId, ResourceExportData exportData); void importResources(TenantId tenantId, List resources); diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index ea70c575fd..cbc03bad51 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -256,6 +256,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic String etag = calculateImageEtag(data); TbResourceInfo existingImage = findSystemOrTenantImageByEtag(tenantId, etag); if (existingImage != null) { + log.info("[{}] Using existing image {}", tenantId, existingImage.getLink()); return new TbResource(existingImage); } } @@ -281,6 +282,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic descriptor.setMediaType(imageData.getMediaType()); image.setDescriptorValue(descriptor); image.setData(data); + log.info("[{}] Creating image {}", tenantId, image.getFileName()); return image; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 2c4304a73c..9cc7febbd6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -59,6 +59,7 @@ import org.thingsboard.server.dao.service.validator.ResourceDataValidator; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -203,6 +204,21 @@ public class BaseResourceService extends AbstractCachedEntityService exportResources(TenantId tenantId, Collection resources) { + return resources.stream() + .map(resourceInfo -> { + if (resourceInfo.getResourceType() == ResourceType.IMAGE) { + ResourceExportData imageExportData = imageService.exportImage(resourceInfo); + imageExportData.setResourceKey(null); // so that the image is not updated by resource key on import + return imageExportData; + } else { + return exportResource(resourceInfo); + } + }) + .toList(); + } + @Override public void importResources(TenantId tenantId, List resources) { for (ResourceExportData resourceData : resources) { @@ -237,17 +253,18 @@ public class BaseResourceService extends AbstractCachedEntityService mapping, Map links) { - AtomicBoolean updated = new AtomicBoolean(false); - processResources(jsonNode, mapping, value -> { + return processResources(jsonNode, mapping, value -> { String link = getResourceLink(value); if (link != null) { String newLink = links.get(link); if (newLink == null || newLink.equals(link)) { return value; // leaving link as is } else { - updated.set(true); return DataConstants.TB_RESOURCE_PREFIX + newLink; } } else { // probably importing an old dashboard json where resources are referenced by ids @@ -435,7 +450,6 @@ public class BaseResourceService extends AbstractCachedEntityService mapping, UnaryOperator processor) { + private boolean processResources(JsonNode jsonNode, Map mapping, UnaryOperator processor) { + AtomicBoolean updated = new AtomicBoolean(false); JacksonUtil.replaceByMapping(jsonNode, mapping, Collections.emptyMap(), (name, urlNode) -> { String value = null; if (urlNode.isTextual()) { // link is in the right place @@ -522,9 +536,13 @@ public class BaseResourceService extends AbstractCachedEntityService