diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java index ce946688ea..ee00b756ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DashboardSyncService.java @@ -80,6 +80,8 @@ public class DashboardSyncService { .map(widgetTypeFile -> getFileContent(widgetTypeFile.path())); widgetsBundleService.updateSystemWidgets(widgetsBundles, widgetTypes); + // TODO: read images folder and save images + RepoFile dashboardFile = listFiles("dashboards").get(0); String dashboardJson = getFileContent(dashboardFile.path()); resourceService.createOrUpdateSystemResource(ResourceType.DASHBOARD, GATEWAYS_DASHBOARD_KEY, dashboardJson); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java index 66794c5ee2..308818d3f5 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java @@ -51,7 +51,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement public Dashboard save(Dashboard dashboard, SecurityUser user) throws Exception { ActionType actionType = dashboard.getId() == null ? ActionType.ADDED : ActionType.UPDATED; TenantId tenantId = dashboard.getTenantId(); - + if (CollectionUtils.isNotEmpty(dashboard.getResources())) { tbResourceService.importResources(dashboard.getResources(), user); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 12431b617a..05bf89009f 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -28,9 +28,13 @@ import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; +import org.thingsboard.server.common.data.ResourceExportData; +import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.TbResourceInfo; import org.thingsboard.server.common.data.alarm.AlarmSeverity; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.RuleNodeId; +import org.thingsboard.server.common.data.id.TbResourceId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.WidgetTypeId; import org.thingsboard.server.common.data.page.PageDataIterable; @@ -41,6 +45,7 @@ import org.thingsboard.server.dao.customer.CustomerDao; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.dashboard.DashboardService; import org.thingsboard.server.dao.device.DeviceConnectivityConfiguration; +import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.resource.ResourceService; import org.thingsboard.server.dao.rule.RuleChainService; import org.thingsboard.server.dao.settings.AdminSettingsService; @@ -51,8 +56,13 @@ import org.thingsboard.server.service.component.ComponentDiscoveryService; import org.thingsboard.server.service.component.RuleNodeClassInfo; import org.thingsboard.server.utils.TbNodeUpgradeUtils; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.concurrent.ExecutionException; @@ -90,6 +100,8 @@ public class DefaultDataUpdateService implements DataUpdateService { @Autowired private ResourceService resourceService; + @Autowired + private ImageService imageService; @Autowired private DashboardService dashboardService; @@ -367,7 +379,7 @@ public class DefaultDataUpdateService implements DataUpdateService { int updatedCount = 0; for (DashboardId dashboardId : dashboards) { Dashboard dashboard = dashboardService.findDashboardById(TenantId.SYS_TENANT_ID, dashboardId); - boolean updated = resourceService.replaceResourcesUsageWithUrls(dashboard); + boolean updated = resourceService.updateResourcesUsage(dashboard); if (updated) { dashboardService.saveDashboard(dashboard); updatedCount++; @@ -386,7 +398,7 @@ public class DefaultDataUpdateService implements DataUpdateService { var widgets = new PageDataIterable<>(widgetTypeService::findAllWidgetTypesIds, 512); for (WidgetTypeId widgetTypeId : widgets) { WidgetTypeDetails widgetTypeDetails = widgetTypeService.findWidgetTypeDetailsById(TenantId.SYS_TENANT_ID, widgetTypeId); - boolean updated = resourceService.replaceResourcesUsageWithUrls(widgetTypeDetails); + boolean updated = resourceService.updateResourcesUsage(widgetTypeDetails); if (updated) { widgetTypeService.saveWidgetType(widgetTypeDetails); updatedCount++; diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java b/application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java index 3914f049ce..0b05b436b3 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java @@ -59,17 +59,18 @@ public class ImagesUpdater { public void updateWidgetTypesImages() { log.info("Updating widget types images..."); var widgetTypesIds = new PageDataIterable<>(widgetTypeDao::findAllWidgetTypesIds, 1024); - updateImages(widgetTypesIds, "widget type", imageService::replaceBase64WithImageUrl, widgetTypeDao); + updateImages(widgetTypesIds, "widget type", imageService::updateImagesUsage, widgetTypeDao); } public void updateDashboardsImages() { log.info("Updating dashboards images..."); - updateImages("dashboard", dashboardDao::findIdsByTenantId, imageService::replaceBase64WithImageUrl, dashboardDao); + updateImages("dashboard", dashboardDao::findIdsByTenantId, imageService::updateImagesUsage, dashboardDao); } public void createSystemImages(Dashboard defaultDashboard) { defaultDashboard.setTenantId(TenantId.SYS_TENANT_ID); - boolean created = imageService.replaceBase64WithImageUrl(defaultDashboard); + // TODO: test! also update dashboards + boolean created = imageService.updateImagesUsage(defaultDashboard); if (created) { log.debug("Created system images for default dashboard '{}'", defaultDashboard.getTitle()); } diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java index 34082d2cc8..641a0b8e4b 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java @@ -43,11 +43,11 @@ import org.thingsboard.server.service.security.permission.AccessControlService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; +import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.Comparator; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -135,41 +135,40 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements @Override public List exportResources(Dashboard dashboard, SecurityUser user) throws ThingsboardException { - return exportResources(dashboard, imageService::inlineImages, resourceService::replaceResourcesUrlsWithTags, user); + return exportResources(dashboard, imageService::getUsedImages, resourceService::getUsedResources, user); } @Override public List exportResources(WidgetTypeDetails widgetTypeDetails, SecurityUser user) throws ThingsboardException { - return exportResources(widgetTypeDetails, imageService::inlineImages, resourceService::replaceResourcesUrlsWithTags, user); + return exportResources(widgetTypeDetails, imageService::getUsedImages, resourceService::getUsedResources, user); } @Override public void importResources(List resources, SecurityUser user) throws Exception { - for (ResourceExportData resourceExportData : resources) { - if (resourceExportData.getType() == ResourceType.IMAGE) { - tbImageService.importImage(resourceExportData, true, user); + for (ResourceExportData resourceData : resources) { + TbResourceInfo resourceInfo; + if (resourceData.getType() == ResourceType.IMAGE) { + resourceInfo = tbImageService.importImage(resourceData, true, user); } else { - importResource(resourceExportData, true, user); + resourceInfo = importResource(resourceData, true, user); } + resourceData.setNewLink(resourceInfo.getLink()); } } private List exportResources(T entity, - Function> imagesProcessor, - Function> resourcesProcessor, + Function> imagesProcessor, + Function> resourcesProcessor, SecurityUser user) throws ThingsboardException { - Map resources = new HashMap<>(); - for (TbResourceInfo imageInfo : imagesProcessor.apply(entity)) { - resources.putIfAbsent(imageInfo.getId(), imageInfo); - } - for (TbResourceInfo resourceInfo : resourcesProcessor.apply(entity)) { - resources.putIfAbsent(resourceInfo.getId(), resourceInfo); - } - for (TbResourceInfo resourceInfo : resources.values()) { + List resources = new ArrayList<>(); + resources.addAll(imagesProcessor.apply(entity)); + resources.addAll(resourcesProcessor.apply(entity)); + + for (TbResourceInfo resourceInfo : resources) { accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resourceInfo.getId(), resourceInfo); } - return resources.values().stream() + return resources.stream() .map(resourceInfo -> { if (resourceInfo.getResourceType() == ResourceType.IMAGE) { ResourceExportData imageExportData = imageService.exportImage(resourceInfo); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java index 7e0bcd51ff..7b0164d65b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DashboardExportService.java @@ -48,8 +48,6 @@ public class DashboardExportService extends BaseEntityExportService ctx, I entityId) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java index c2d78791da..587ff82e4b 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/WidgetTypeExportService.java @@ -36,8 +36,6 @@ public class WidgetTypeExportService extends BaseEntityExportService> resources = exportedDashboard.getResources().stream() .collect(Collectors.groupingBy(ResourceExportData::getType)); assertThat(resources.get(ResourceType.IMAGE)).singleElement().satisfies(exportedImage -> { assertThat(exportedImage.getFileName()).isEqualTo(imageInfo.getResourceKey()); assertThat(exportedImage.getData()).isEqualTo(Base64.getEncoder().encodeToString(ImageControllerTest.PNG_IMAGE)); - assertThat(exportedImage.getEtag()).isEqualTo(imageInfo.getEtag()); }); assertThat(resources.get(ResourceType.JS_MODULE)).singleElement().satisfies(exportedJsModule -> { - assertThat(exportedJsModule.getFileName()).isEqualTo(resource.getResourceKey()); + assertThat(exportedJsModule.getFileName()).isEqualTo(resourceInfo.getResourceKey()); assertThat(exportedJsModule.getData()).isEqualTo(Base64.getEncoder().encodeToString(resourceData)); - assertThat(exportedJsModule.getEtag()).isEqualTo(resourceInfo.getEtag()); }); doDelete("/api/dashboard/" + dashboard.getId()).andExpect(status().isOk()); doDelete("/api/images/tenant/" + imageInfo.getResourceKey()).andExpect(status().isOk()); - doDelete("/api/resource/" + resourceInfo.getId()).andExpect(status().isOk()); + resource = new TbResource(resourceInfo); + resource.setData(new byte[]{1, 2, 3}); // updating resource data to check that a new resource will be created + doPost("/api/resource", resource, TbResourceInfo.class); Dashboard importedDashboard = doPost("/api/dashboard", exportedDashboard, Dashboard.class); - assertThat(importedDashboard.getConfiguration().get("someImage").asText()).isEqualTo("tb-image;/api/images/tenant/" + imageInfo.getResourceKey()); + imageRef = importedDashboard.getConfiguration().get("someImage").asText(); + assertThat(imageRef).isEqualTo("tb-image;/api/images/tenant/" + imageInfo.getResourceKey()); + resourceRef = importedDashboard.getConfiguration().get("widgets").get("xxx").get("config") + .get("actions").get("elementClick").get(0).get("customResources").get(0).get("url").asText(); + String newResourceKey = "gateway-management-extension_(1).js"; + assertThat(resourceRef).isEqualTo("tb-resource;/api/resource/js_module/tenant/" + newResourceKey); TbResourceInfo importedImageInfo = doGet("/api/images/tenant/" + imageInfo.getResourceKey() + "/info", TbResourceInfo.class); assertThat(importedImageInfo.getEtag()).isEqualTo(imageInfo.getEtag()); assertThat(importedImageInfo.getResourceKey()).isEqualTo(imageInfo.getResourceKey()); - TbResourceInfo importedResourceInfo = doGet(resourceInfo.getLink() + "/info", TbResourceInfo.class); + TbResourceInfo importedResourceInfo = doGet("/api/resource/js_module/tenant/" + newResourceKey + "/info", TbResourceInfo.class); assertThat(importedResourceInfo.getEtag()).isEqualTo(resourceInfo.getEtag()); - assertThat(importedResourceInfo.getResourceKey()).isEqualTo(resourceInfo.getResourceKey()); } private Dashboard createDashboard(String title) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java index abc7fbfaf3..c3459bf84f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java @@ -28,7 +28,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.widget.WidgetTypeDetails; -import java.util.List; +import java.util.Collection; public interface ImageService { @@ -58,15 +58,15 @@ public interface ImageService { boolean replaceBase64WithImageUrl(HasImage entity, String type); - boolean replaceBase64WithImageUrl(Dashboard dashboard); + boolean updateImagesUsage(Dashboard dashboard); - boolean replaceBase64WithImageUrl(WidgetTypeDetails widgetType); + boolean updateImagesUsage(WidgetTypeDetails widgetType); void inlineImage(HasImage entity); - List inlineImages(Dashboard dashboard); + Collection getUsedImages(Dashboard dashboard); - List inlineImages(WidgetTypeDetails widgetTypeDetails); + Collection getUsedImages(WidgetTypeDetails widgetTypeDetails); void inlineImageForEdge(HasImage entity); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java index 3f62136ef4..89ba6667d4 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java @@ -73,13 +73,13 @@ public interface ResourceService extends EntityDaoService { TbResourceInfo findSystemOrTenantResourceByEtag(TenantId tenantId, ResourceType resourceType, String etag); - boolean replaceResourcesUsageWithUrls(Dashboard dashboard); + boolean updateResourcesUsage(Dashboard dashboard); - boolean replaceResourcesUsageWithUrls(WidgetTypeDetails widgetTypeDetails); + boolean updateResourcesUsage(WidgetTypeDetails widgetTypeDetails); - List replaceResourcesUrlsWithTags(Dashboard dashboard); + List getUsedResources(Dashboard dashboard); - List replaceResourcesUrlsWithTags(WidgetTypeDetails widgetTypeDetails); + List getUsedResources(WidgetTypeDetails widgetTypeDetails); TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, String data); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ResourceExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/ResourceExportData.java index 9d9cb538ad..b873c3dc8e 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ResourceExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ResourceExportData.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.common.data; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import io.swagger.v3.oas.annotations.media.Schema; import lombok.AllArgsConstructor; @@ -32,7 +33,7 @@ import lombok.extern.slf4j.Slf4j; @JsonInclude(JsonInclude.Include.NON_NULL) public class ResourceExportData { - private String etag; + private String link; private String title; private ResourceType type; private ResourceSubType subType; @@ -43,4 +44,11 @@ public class ResourceExportData { private String mediaType; private String data; + /* + * when importing resource, the previous link may be changed due to existing duplicates or something else. + * this is the new proper link to be used in place of the old link + * */ + @JsonIgnore + private String newLink; + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java index 96db46c1d1..ff5ddc27e7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java @@ -161,8 +161,8 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb dashboardValidator.validate(dashboard, DashboardInfo::getTenantId); } try { - imageService.replaceBase64WithImageUrl(dashboard); - resourceService.replaceResourcesUsageWithUrls(dashboard); + imageService.updateImagesUsage(dashboard); + resourceService.updateResourcesUsage(dashboard); var saved = dashboardDao.save(dashboard.getTenantId(), dashboard); publishEvictEvent(new DashboardTitleEvictEvent(saved.getId())); diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index 44046a92ad..9097d8372b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.resource; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import jakarta.annotation.PostConstruct; import lombok.Data; import lombok.SneakyThrows; @@ -57,8 +56,8 @@ import org.thingsboard.server.dao.util.ImageUtils.ProcessedImage; import org.thingsboard.server.dao.widget.WidgetTypeDao; import org.thingsboard.server.dao.widget.WidgetsBundleDao; -import java.util.ArrayList; import java.util.Base64; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -236,6 +235,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic ImageDescriptor descriptor = imageInfo.getDescriptor(ImageDescriptor.class); byte[] data = getImageData(imageInfo.getTenantId(), imageInfo.getId()); return ResourceExportData.builder() + .link(imageInfo.getLink()) .mediaType(descriptor.getMediaType()) .fileName(imageInfo.getFileName()) .title(imageInfo.getTitle()) @@ -245,7 +245,6 @@ public class BaseImageService extends BaseResourceService implements ImageServic .isPublic(imageInfo.isPublic()) .publicResourceKey(imageInfo.getPublicResourceKey()) .data(Base64.getEncoder().encodeToString(data)) - .etag(descriptor.getEtag()) .build(); } @@ -298,51 +297,59 @@ public class BaseImageService extends BaseResourceService implements ImageServic } imageName = imageName + type + " image"; - UpdateResult result = base64ToImageUrl(entity.getTenantId(), imageName, entity.getImage()); + UpdateResult result = convertToImageUrl(entity.getTenantId(), imageName, entity.getImage(), Collections.emptyMap()); entity.setImage(result.getValue()); return result.isUpdated(); } @Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure @Override - public boolean replaceBase64WithImageUrl(WidgetTypeDetails entity) { - log.trace("Executing replaceBase64WithImageUrl [{}] [WidgetTypeDetails] [{}]", entity.getTenantId(), entity.getId()); - String prefix = "\"" + entity.getName() + "\" "; - if (entity.getTenantId() == null || entity.getTenantId().isSysTenantId()) { + public boolean updateImagesUsage(WidgetTypeDetails widgetTypeDetails) { + TenantId tenantId = widgetTypeDetails.getTenantId(); + log.trace("Executing updateImagesUsage [{}] [WidgetTypeDetails] [{}]", tenantId, widgetTypeDetails.getId()); + String prefix = "\"" + widgetTypeDetails.getName() + "\" "; + if (tenantId == null || tenantId.isSysTenantId()) { prefix += "system "; } prefix += "widget"; - UpdateResult result = base64ToImageUrl(entity.getTenantId(), prefix + " image", entity.getImage()); - entity.setImage(result.getValue()); + Map imagesLinks = getResourcesLinks(widgetTypeDetails.getResources()); + + UpdateResult result = convertToImageUrl(tenantId, prefix + " image", widgetTypeDetails.getImage(), imagesLinks); boolean updated = result.isUpdated(); - if (entity.getDescriptor().isObject()) { - JsonNode defaultConfig = entity.getDefaultConfig(); + widgetTypeDetails.setImage(result.getValue()); + + if (widgetTypeDetails.getDescriptor().isObject()) { + JsonNode defaultConfig = widgetTypeDetails.getDefaultConfig(); if (defaultConfig != null) { - updated |= base64ToImageUrlUsingMapping(entity.getTenantId(), WIDGET_TYPE_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), defaultConfig); - entity.setDefaultConfig(defaultConfig); + updated |= convertToImageUrlsByMapping(tenantId, WIDGET_TYPE_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), defaultConfig, imagesLinks); + widgetTypeDetails.setDefaultConfig(defaultConfig); } } - updated |= base64ToImageUrlRecursively(entity.getTenantId(), prefix, entity.getDescriptor()); + updated |= convertToImageUrls(tenantId, prefix, widgetTypeDetails.getDescriptor(), imagesLinks); return updated; } @Transactional(noRollbackFor = Exception.class) // we don't want transaction to rollback in case of an image processing failure @Override - public boolean replaceBase64WithImageUrl(Dashboard entity) { - log.trace("Executing replaceBase64WithImageUrl [{}] [Dashboard] [{}]", entity.getTenantId(), entity.getId()); - String prefix = "\"" + entity.getTitle() + "\" dashboard"; - var result = base64ToImageUrl(entity.getTenantId(), prefix + " image", entity.getImage()); + public boolean updateImagesUsage(Dashboard dashboard) { + TenantId tenantId = dashboard.getTenantId(); + log.trace("Executing updateImagesUsage [{}] [Dashboard] [{}]", tenantId, dashboard.getId()); + String prefix = "\"" + dashboard.getTitle() + "\" dashboard"; + Map imagesLinks = getResourcesLinks(dashboard.getResources()); + + var result = convertToImageUrl(tenantId, prefix + " image", dashboard.getImage(), imagesLinks); boolean updated = result.isUpdated(); - entity.setImage(result.getValue()); - updated |= base64ToImageUrlUsingMapping(entity.getTenantId(), DASHBOARD_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), entity.getConfiguration()); - updated |= base64ToImageUrlRecursively(entity.getTenantId(), prefix, entity.getConfiguration()); + dashboard.setImage(result.getValue()); + + updated |= convertToImageUrlsByMapping(tenantId, DASHBOARD_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), dashboard.getConfiguration(), imagesLinks); + updated |= convertToImageUrls(tenantId, prefix, dashboard.getConfiguration(), imagesLinks); return updated; } - private boolean base64ToImageUrlUsingMapping(TenantId tenantId, Map mapping, Map templateParams, JsonNode configuration) { + private boolean convertToImageUrlsByMapping(TenantId tenantId, Map mapping, Map templateParams, JsonNode configuration, Map links) { AtomicBoolean updated = new AtomicBoolean(false); JacksonUtil.replaceAllByMapping(configuration, mapping, templateParams, (name, value) -> { - UpdateResult result = base64ToImageUrl(tenantId, name, value); + UpdateResult result = convertToImageUrl(tenantId, name, value, links); if (result.isUpdated()) { updated.set(true); } @@ -351,22 +358,32 @@ public class BaseImageService extends BaseResourceService implements ImageServic return updated.get(); } - private UpdateResult base64ToImageUrl(TenantId tenantId, String name, String data) { - return base64ToImageUrl(tenantId, name, data, false); + private UpdateResult convertToImageUrl(TenantId tenantId, String name, String data, Map links) { + return convertToImageUrl(tenantId, name, data, false, links); } public static final Pattern TB_IMAGE_METADATA_PATTERN = Pattern.compile("^tb-image:([^;]+);data:(.*);.*"); - private UpdateResult base64ToImageUrl(TenantId tenantId, String name, String data, boolean strict) { + private UpdateResult convertToImageUrl(TenantId tenantId, String name, String data, boolean strict, Map imagesLinks) { if (StringUtils.isBlank(data)) { return UpdateResult.of(false, data); } + String link = getImageLink(data); + if (link != null) { + String newLink = imagesLinks.get(link); + if (newLink == null || newLink.equals(link)) { + return UpdateResult.of(false, data); + } else { + return UpdateResult.of(true, DataConstants.TB_IMAGE_PREFIX + newLink); + } + } + String resourceKey = null; String resourceName = null; String resourceSubType = null; String etag = null; - String mediaType = null; + String mediaType; var matcher = TB_IMAGE_METADATA_PATTERN.matcher(data); if (matcher.matches()) { String[] metadata = matcher.group(1).split(":"); @@ -375,8 +392,6 @@ public class BaseImageService extends BaseResourceService implements ImageServic resourceSubType = decode(get(metadata, 2)); etag = get(metadata, 3); mediaType = matcher.group(2); - } else if (data.startsWith("tb-image:")) { - etag = StringUtils.substringAfter(data, "tb-image:"); } else if (data.startsWith(DataConstants.TB_IMAGE_PREFIX + "data:image/") || (!strict && data.startsWith("data:image/"))) { mediaType = StringUtils.substringBetween(data, "data:", ";base64"); } else { @@ -386,9 +401,6 @@ public class BaseImageService extends BaseResourceService implements ImageServic String base64Data = StringUtils.substringAfter(data, "base64,"); byte[] imageData = StringUtils.isNotEmpty(base64Data) ? Base64.getDecoder().decode(base64Data) : null; if (StringUtils.isBlank(etag)) { - if (imageData == null) { - return UpdateResult.of(false, data); - } etag = calculateEtag(imageData); } var imageInfo = findSystemOrTenantImageByEtag(tenantId, etag); @@ -439,10 +451,10 @@ public class BaseImageService extends BaseResourceService implements ImageServic return UpdateResult.of(true, DataConstants.TB_IMAGE_PREFIX + imageInfo.getLink()); } - private boolean base64ToImageUrlRecursively(TenantId tenantId, String title, JsonNode root) { + private boolean convertToImageUrls(TenantId tenantId, String title, JsonNode root, Map links) { AtomicBoolean updated = new AtomicBoolean(false); JacksonUtil.replaceAll(root, title, (path, value) -> { - UpdateResult result = base64ToImageUrl(tenantId, path, value, true); + UpdateResult result = convertToImageUrl(tenantId, path, value, true, links); if (result.isUpdated()) { updated.set(true); } @@ -454,101 +466,116 @@ public class BaseImageService extends BaseResourceService implements ImageServic @Override public void inlineImage(HasImage entity) { log.trace("Executing inlineImage [{}] [{}] [{}]", entity.getTenantId(), entity.getClass().getSimpleName(), entity.getName()); - inlineImage(entity, null); + entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage(), true)); } @Override - public List inlineImages(Dashboard dashboard) { - log.trace("Executing inlineImage [{}] [Dashboard] [{}]", dashboard.getTenantId(), dashboard.getId()); - List images = new ArrayList<>(); - inlineImage(dashboard, images); - inlineIntoJson(dashboard.getTenantId(), dashboard.getConfiguration(), images); - return images; + public Collection getUsedImages(Dashboard dashboard) { + TenantId tenantId = dashboard.getTenantId(); + log.trace("Executing getUsedImages [{}] [Dashboard] [{}]", tenantId, dashboard.getId()); + Map images = new HashMap<>(); + processImage(tenantId, "image", dashboard.getImage(), (key, imageInfo) -> { + images.putIfAbsent(imageInfo.getId(), imageInfo); + return null; // leaving the url as is + }); + processImages(tenantId, dashboard.getConfiguration(), (key, imageInfo) -> { + images.putIfAbsent(imageInfo.getId(), imageInfo); + return null; // leaving the url as is + }); + return images.values(); } @Override - public List inlineImages(WidgetTypeDetails widgetTypeDetails) { - log.trace("Executing inlineImage [{}] [WidgetTypeDetails] [{}]", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId()); - List images = new ArrayList<>(); - inlineImage(widgetTypeDetails, images); - ObjectNode descriptor = (ObjectNode) widgetTypeDetails.getDescriptor(); - inlineIntoJson(widgetTypeDetails.getTenantId(), descriptor, images); + public Collection getUsedImages(WidgetTypeDetails widgetTypeDetails) { + TenantId tenantId = widgetTypeDetails.getTenantId(); + log.trace("Executing getUsedImages [{}] [WidgetTypeDetails] [{}]", tenantId, widgetTypeDetails.getId()); + Map images = new HashMap<>(); + processImage(tenantId, "image", widgetTypeDetails.getImage(), (key, imageInfo) -> { + images.putIfAbsent(imageInfo.getId(), imageInfo); + return null; // leaving the url as is + }); + processImages(tenantId, widgetTypeDetails.getDescriptor(), (key, imageInfo) -> { + images.putIfAbsent(imageInfo.getId(), imageInfo); + return null; // leaving the url as is + }); JsonNode defaultConfig = widgetTypeDetails.getDefaultConfig(); if (defaultConfig != null) { - inlineIntoJson(widgetTypeDetails.getTenantId(), defaultConfig, images); - widgetTypeDetails.setDefaultConfig(defaultConfig); + processImages(tenantId, defaultConfig, (key, imageInfo) -> { + images.putIfAbsent(imageInfo.getId(), imageInfo); + return null; // leaving the url as is + }); } - return images; + return images.values(); } @Override public void inlineImageForEdge(HasImage entity) { log.trace("Executing inlineImageForEdge [{}] [{}] [{}]", entity.getTenantId(), entity.getClass().getSimpleName(), entity.getName()); - entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage(), false, null)); + entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage(), false)); } @Override public void inlineImagesForEdge(Dashboard dashboard) { log.trace("Executing inlineImagesForEdge [{}] [Dashboard] [{}]", dashboard.getTenantId(), dashboard.getId()); inlineImageForEdge(dashboard); - inlineIntoJson(dashboard.getTenantId(), dashboard.getConfiguration(), false, null); + inlineImages(dashboard.getTenantId(), dashboard.getConfiguration(), false); } @Override public void inlineImagesForEdge(WidgetTypeDetails widgetTypeDetails) { log.trace("Executing inlineImage [{}] [WidgetTypeDetails] [{}]", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId()); inlineImageForEdge(widgetTypeDetails); - inlineIntoJson(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), false, null); + inlineImages(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), false); } - private void inlineImage(HasImage entity, List processedImages) { - log.trace("Executing inlineImage [{}] [{}] [{}]", entity.getTenantId(), entity.getClass().getSimpleName(), entity.getName()); - entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage(), true, processedImages)); - } - - private void inlineIntoJson(TenantId tenantId, JsonNode root, List processedImages) { - inlineIntoJson(tenantId, root, true, processedImages); + private void inlineImages(TenantId tenantId, JsonNode root, boolean addTbImagePrefix) { + processImages(tenantId, root, (key, imageInfo) -> { + return inlineImage(key, imageInfo, addTbImagePrefix); + }); } - private void inlineIntoJson(TenantId tenantId, JsonNode root, boolean addTbImagePrefix, List processedImages) { - JacksonUtil.replaceAll(root, "", (path, value) -> inlineImage(tenantId, path, value, addTbImagePrefix, processedImages)); + private String inlineImage(TenantId tenantId, String path, String url, boolean addTbImagePrefix) { + return processImage(tenantId, path, url, (key, imageInfo) -> { + return inlineImage(key, imageInfo, addTbImagePrefix); + }); } - private String inlineImage(TenantId tenantId, String path, String url, boolean addTbImagePrefix, List processedImages) { - return inlineImage(tenantId, path, url, (key, imageInfo) -> { - ImageDescriptor descriptor = getImageDescriptor(imageInfo, key.isPreview()); - String value = ""; - if (addTbImagePrefix) { - value = "tb-image:"; - - if (processedImages != null && !key.isPreview()) { // images are stored separately - processedImages.add(imageInfo); - value += descriptor.getEtag(); - return value; - } + private String inlineImage(ImageCacheKey key, TbResourceInfo imageInfo, boolean addTbImagePrefix) { + String value = ""; + if (addTbImagePrefix) { + value = "tb-image:" + encode(imageInfo.getResourceKey()) + ":" + + encode(imageInfo.getName()) + ":" + + encode(imageInfo.getResourceSubType().name()) + ";"; + } - value += encode(imageInfo.getResourceKey()) + ":" - + encode(imageInfo.getName()) + ":" - + encode(imageInfo.getResourceSubType().name()) + ":" - + imageInfo.getEtag() + ";"; - } + ImageDescriptor descriptor = getImageDescriptor(imageInfo, key.isPreview()); + byte[] data = key.isPreview() ? getImagePreview(key.getTenantId(), imageInfo.getId()) : getImageData(key.getTenantId(), imageInfo.getId()); + return value + "data:" + descriptor.getMediaType() + ";base64," + encode(data); + } - byte[] data = key.isPreview() ? getImagePreview(tenantId, imageInfo.getId()) : getImageData(tenantId, imageInfo.getId()); - return value + "data:" + descriptor.getMediaType() + ";base64," + encode(data); + private void processImages(TenantId tenantId, JsonNode node, BiFunction processor) { + JacksonUtil.replaceAll(node, "", (path, value) -> { + return processImage(tenantId, path, value, processor); }); } - private String inlineImage(TenantId tenantId, String path, String imageUrl, BiFunction inliner) { + private String processImage(TenantId tenantId, String path, String imageUrl, BiFunction processor) { try { ImageCacheKey key = getKeyFromUrl(tenantId, imageUrl); if (key != null) { var imageInfo = getImageInfoByTenantIdAndKey(key.getTenantId(), key.getResourceKey()); - if (imageInfo != null && !(TenantId.SYS_TENANT_ID.equals(imageInfo.getTenantId()) && ResourceSubType.SCADA_SYMBOL.equals(imageInfo.getResourceSubType()))) { - return inliner.apply(key, imageInfo); + // TODO: maybe export scada too? + if (imageInfo == null || (TenantId.SYS_TENANT_ID.equals(imageInfo.getTenantId()) && ResourceSubType.SCADA_SYMBOL.equals(imageInfo.getResourceSubType()))) { + return imageUrl; + } else { + String result = processor.apply(key, imageInfo); + if (result != null) { + return result; + } } } } catch (Exception e) { - log.warn("[{}][{}][{}] Failed to inline image.", tenantId, path, imageUrl, e); + log.warn("[{}][{}][{}] Failed to process image", tenantId, path, imageUrl, e); } return imageUrl; } @@ -562,10 +589,15 @@ public class BaseImageService extends BaseResourceService implements ImageServic if (StringUtils.isBlank(url)) { return null; } + String link = getImageLink(url); + if (link == null) { + return null; + } + TenantId imageTenantId = null; - if (url.startsWith(DataConstants.TB_IMAGE_PREFIX + "/api/images/tenant/")) { + if (link.startsWith("/api/images/tenant/")) { imageTenantId = tenantId; - } else if (url.startsWith(DataConstants.TB_IMAGE_PREFIX + "/api/images/system/")) { + } else if (link.startsWith("/api/images/system/")) { imageTenantId = TenantId.SYS_TENANT_ID; } if (imageTenantId != null) { @@ -579,10 +611,19 @@ public class BaseImageService extends BaseResourceService implements ImageServic return null; } + private String getImageLink(String value) { + if (value.startsWith(DataConstants.TB_IMAGE_PREFIX + "/api/images")) { + return StringUtils.removeStart(value, DataConstants.TB_IMAGE_PREFIX); + } else { + return null; + } + } + @Data(staticConstructor = "of") private static class UpdateResult { private final boolean updated; private final String value; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java index 146be64a54..0e63aea523 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java @@ -21,6 +21,7 @@ import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.hibernate.exception.ConstraintViolationException; import org.springframework.beans.factory.annotation.Autowired; @@ -58,6 +59,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Base64; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -188,6 +190,7 @@ public class BaseResourceService extends AbstractCachedEntityService links = getResourcesLinks(dashboard.getResources()); + return updateResourcesUsage(dashboard.getTenantId(), dashboard.getConfiguration(), DASHBOARD_RESOURCES_MAPPING, links); } @Override - public boolean replaceResourcesUsageWithUrls(WidgetTypeDetails widgetTypeDetails) { - boolean updated = replaceResourcesUsageWithUrls(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), WIDGET_RESOURCES_MAPPING); + public boolean updateResourcesUsage(WidgetTypeDetails widgetTypeDetails) { + Map links = getResourcesLinks(widgetTypeDetails.getResources()); + boolean updated = updateResourcesUsage(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), WIDGET_RESOURCES_MAPPING, links); JsonNode defaultConfig = widgetTypeDetails.getDefaultConfig(); if (defaultConfig != null) { - updated |= replaceResourcesUsageWithUrls(widgetTypeDetails.getTenantId(), defaultConfig, WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING); + updated |= updateResourcesUsage(widgetTypeDetails.getTenantId(), defaultConfig, WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING, links); widgetTypeDetails.setDefaultConfig(defaultConfig); } return updated; } - private boolean replaceResourcesUsageWithUrls(TenantId tenantId, JsonNode jsonNode, Map mapping) { - AtomicBoolean updated = new AtomicBoolean(false); - processResources(jsonNode, mapping, value -> { - if (value.startsWith(DataConstants.TB_RESOURCE_PREFIX + "/api/resource")) { // already a link, ignoring - return value; - } - - TbResourceInfo resourceInfo; - if (value.startsWith("tb-resource:")) { // tag with metadata, probably importing - String[] metadata = StringUtils.removeStart(value, "tb-resource:").split(":"); - if (metadata.length < 2) { - return value; + protected Map getResourcesLinks(List resources) { + Map links; + if (CollectionUtils.isNotEmpty(resources)) { + links = new HashMap<>(); + resources.forEach(resource -> { + if (resource.getNewLink() != null) { + links.put(resource.getLink(), resource.getNewLink()); } - ResourceType resourceType = ResourceType.valueOf(metadata[0]); - String etag = metadata[1]; + }); + } else { + links = Collections.emptyMap(); + } + return links; + } - resourceInfo = findSystemOrTenantResourceByEtag(tenantId, resourceType, etag); - if (resourceInfo == null) { - log.warn("[{}] Couldn't find resource referenced as '{}'", tenantId, value); - return value; + private boolean updateResourcesUsage(TenantId tenantId, JsonNode jsonNode, Map mapping, Map links) { + AtomicBoolean updated = new AtomicBoolean(false); + processResources(jsonNode, mapping, value -> { + String link = getResourceLink(value); + if (link != null) { + String newLink = links.get(link); + if (newLink == null || newLink.equals(link)) { + return value; // leaving link as is + } else { + updated.set(true); + return DataConstants.TB_RESOURCE_PREFIX + newLink; } } else { // probably importing an old dashboard json where resources are referenced by ids TbResourceId resourceId; @@ -354,40 +364,39 @@ public class BaseResourceService extends AbstractCachedEntityService replaceResourcesUrlsWithTags(Dashboard dashboard) { - return replaceResourcesUrlsWithTags(dashboard.getTenantId(), dashboard.getConfiguration(), DASHBOARD_RESOURCES_MAPPING); + public List getUsedResources(Dashboard dashboard) { + return getUsedResources(dashboard.getTenantId(), dashboard.getConfiguration(), DASHBOARD_RESOURCES_MAPPING); } @Override - public List replaceResourcesUrlsWithTags(WidgetTypeDetails widgetTypeDetails) { - List resources = replaceResourcesUrlsWithTags(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), WIDGET_RESOURCES_MAPPING); + public List getUsedResources(WidgetTypeDetails widgetTypeDetails) { + List resources = getUsedResources(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), WIDGET_RESOURCES_MAPPING); JsonNode defaultConfig = widgetTypeDetails.getDefaultConfig(); if (defaultConfig != null) { - resources.addAll(replaceResourcesUrlsWithTags(widgetTypeDetails.getTenantId(), defaultConfig, WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING)); - widgetTypeDetails.setDefaultConfig(defaultConfig); + resources.addAll(getUsedResources(widgetTypeDetails.getTenantId(), defaultConfig, WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING)); } return resources; } - private List replaceResourcesUrlsWithTags(TenantId tenantId, JsonNode jsonNode, Map mapping) { + private List getUsedResources(TenantId tenantId, JsonNode jsonNode, Map mapping) { List resources = new ArrayList<>(); processResources(jsonNode, mapping, value -> { - if (!value.startsWith(DataConstants.TB_RESOURCE_PREFIX + "/api/resource/")) { + String link = getResourceLink(value); + if (link == null) { return value; } @@ -395,27 +404,35 @@ public class BaseResourceService extends AbstractCachedEntityService mapping, UnaryOperator processor) { JacksonUtil.replaceByMapping(jsonNode, mapping, Collections.emptyMap(), (name, urlNode) -> { String value = null; @@ -446,8 +463,8 @@ public class BaseResourceService extends AbstractCachedEntityService