Merge pull request #12174 from thingsboard/feature/new-widget-resources-export

Dashboard/widget resources export - process whole config

application/src/main/java/org/thingsboard/server/service/install/update/ResourcesUpdater.java

@@ -101,7 +101,7 @@ public class ResourcesUpdater {
         for (DashboardId dashboardId : dashboards) {
             executor.submit(() -> {
                 Dashboard dashboard = dashboardService.findDashboardById(TenantId.SYS_TENANT_ID, dashboardId);
-                boolean updated = resourceService.updateResourcesUsage(dashboard); // will convert resources ids to new structure
+                boolean updated = resourceService.updateResourcesUsage(dashboard.getTenantId(), dashboard); // will convert resources ids to new structure
                 if (updated) {
                     dashboardService.saveDashboard(dashboard);
                     updatedCount.incrementAndGet();
@@ -130,7 +130,7 @@ public class ResourcesUpdater {
         for (WidgetTypeId widgetTypeId : widgets) {
             executor.submit(() -> {
                 WidgetTypeDetails widgetTypeDetails = widgetTypeService.findWidgetTypeDetailsById(TenantId.SYS_TENANT_ID, widgetTypeId);
-                boolean updated = resourceService.updateResourcesUsage(widgetTypeDetails);
+                boolean updated = resourceService.updateResourcesUsage(widgetTypeDetails.getTenantId(), widgetTypeDetails);
                 if (updated) {
                     widgetTypeService.saveWidgetType(widgetTypeDetails);
                     updatedCount.incrementAndGet();

application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java

@@ -46,7 +46,7 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Comparator;
 import java.util.List;
-import java.util.function.Function;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -132,12 +132,12 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
 
     @Override
     public List<ResourceExportData> exportResources(Dashboard dashboard, SecurityUser user) throws ThingsboardException {
-        return exportResources(dashboard, imageService::getUsedImages, resourceService::getUsedResources, user);
+        return exportResources(() -> imageService.getUsedImages(dashboard), () -> resourceService.getUsedResources(user.getTenantId(), dashboard), user);
     }
 
     @Override
     public List<ResourceExportData> exportResources(WidgetTypeDetails widgetTypeDetails, SecurityUser user) throws ThingsboardException {
-        return exportResources(widgetTypeDetails, imageService::getUsedImages, resourceService::getUsedResources, user);
+        return exportResources(() -> imageService.getUsedImages(widgetTypeDetails), () -> resourceService.getUsedResources(user.getTenantId(), widgetTypeDetails), user);
     }
 
     @Override
@@ -153,13 +153,12 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
         }
     }
 
-    private <T> List<ResourceExportData> exportResources(T entity,
-                                                         Function<T, Collection<TbResourceInfo>> imagesProcessor,
-                                                         Function<T, Collection<TbResourceInfo>> resourcesProcessor,
+    private <T> List<ResourceExportData> exportResources(Supplier<Collection<TbResourceInfo>> imagesProcessor,
+                                                         Supplier<Collection<TbResourceInfo>> resourcesProcessor,
                                                          SecurityUser user) throws ThingsboardException {
         List<TbResourceInfo> resources = new ArrayList<>();
-        resources.addAll(imagesProcessor.apply(entity));
-        resources.addAll(resourcesProcessor.apply(entity));
+        resources.addAll(imagesProcessor.get());
+        resources.addAll(resourcesProcessor.get());
         for (TbResourceInfo resourceInfo : resources) {
             accessControlService.checkPermission(user, Resource.TB_RESOURCE, Operation.READ, resourceInfo.getId(), resourceInfo);
         }

application/src/test/java/org/thingsboard/server/controller/DashboardControllerTest.java

@@ -17,6 +17,7 @@ package org.thingsboard.server.controller;
 
 import com.datastax.oss.driver.api.core.uuid.Uuids;
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -602,13 +603,14 @@ public class DashboardControllerTest extends AbstractControllerTest {
         dashboard.setTitle("My dashboard");
         dashboard.setConfiguration(JacksonUtil.newObjectNode()
                 .put("someImage", "tb-image;/api/images/tenant/" + imageInfo.getResourceKey())
-                .set("widgets", JacksonUtil.toJsonNode("""
+                .<ObjectNode>set("widgets", JacksonUtil.toJsonNode("""
                         {"xxx":
                         {"config":{"actions":{"elementClick":[
                         {"customResources":[{"url":{"entityType":"TB_RESOURCE","id":
                         "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"},"isModule":true},
                         {"url":"tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js","isModule":true}]}]}}}}
-                        """)));
+                        """))
+                .put("someResource", "tb-resource;/api/resource/js_module/tenant/gateway-management-extension.js"));
         dashboard = doPost("/api/dashboard", dashboard, Dashboard.class);
 
         Dashboard exportedDashboard = doGet("/api/dashboard/" + dashboard.getUuidId() + "?includeResources=true", Dashboard.class);
@@ -637,12 +639,18 @@ public class DashboardControllerTest extends AbstractControllerTest {
         doPost("/api/resource", resource, TbResourceInfo.class);
 
         Dashboard importedDashboard = doPost("/api/dashboard", exportedDashboard, Dashboard.class);
+        String newResourceKey = "gateway-management-extension_(1).js";
+
         imageRef = importedDashboard.getConfiguration().get("someImage").asText();
         assertThat(imageRef).isEqualTo("tb-image;/api/images/tenant/" + imageInfo.getResourceKey());
-        resourceRef = importedDashboard.getConfiguration().get("widgets").get("xxx").get("config")
-                .get("actions").get("elementClick").get(0).get("customResources").get(0).get("url").asText();
-        String newResourceKey = "gateway-management-extension_(1).js";
-        assertThat(resourceRef).isEqualTo("tb-resource;/api/resource/js_module/tenant/" + newResourceKey);
+
+        List<String> resourcesRefs = new ArrayList<>();
+        resourcesRefs.add(importedDashboard.getConfiguration().get("widgets").get("xxx").get("config")
+                .get("actions").get("elementClick").get(0).get("customResources").get(0).get("url").asText());
+        resourcesRefs.add(importedDashboard.getConfiguration().get("someResource").asText());
+        assertThat(resourcesRefs).allSatisfy(ref -> {
+            assertThat(ref).isEqualTo("tb-resource;/api/resource/js_module/tenant/" + newResourceKey);
+        });
 
         TbResourceInfo importedImageInfo = doGet("/api/images/tenant/" + imageInfo.getResourceKey() + "/info", TbResourceInfo.class);
         assertThat(importedImageInfo.getEtag()).isEqualTo(imageInfo.getEtag());

common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ResourceService.java

@@ -80,13 +80,13 @@ public interface ResourceService extends EntityDaoService {
 
     TbResourceInfo findSystemOrTenantResourceByEtag(TenantId tenantId, ResourceType resourceType, String etag);
 
-    boolean updateResourcesUsage(Dashboard dashboard);
+    boolean updateResourcesUsage(TenantId tenantId, Dashboard dashboard);
 
-    boolean updateResourcesUsage(WidgetTypeDetails widgetTypeDetails);
+    boolean updateResourcesUsage(TenantId tenantId, WidgetTypeDetails widgetTypeDetails);
 
-    Collection<TbResourceInfo> getUsedResources(Dashboard dashboard);
+    Collection<TbResourceInfo> getUsedResources(TenantId tenantId, Dashboard dashboard);
 
-    Collection<TbResourceInfo> getUsedResources(WidgetTypeDetails widgetTypeDetails);
+    Collection<TbResourceInfo> getUsedResources(TenantId tenantId, WidgetTypeDetails widgetTypeDetails);
 
     TbResource createOrUpdateSystemResource(ResourceType resourceType, String resourceKey, byte[] data);
 

common/data/src/main/java/org/thingsboard/server/common/data/widget/WidgetType.java

@@ -50,7 +50,8 @@ public class WidgetType extends BaseWidgetType {
 
     @JsonIgnore
     public JsonNode getDefaultConfig() {
-        return Optional.ofNullable(descriptor.get("defaultConfig"))
+        return Optional.ofNullable(descriptor)
+                .map(descriptor -> descriptor.get("defaultConfig"))
                 .filter(JsonNode::isTextual).map(JsonNode::asText)
                 .map(json -> {
                     try {

dao/src/main/java/org/thingsboard/server/dao/dashboard/DashboardServiceImpl.java

@@ -162,18 +162,19 @@ public class DashboardServiceImpl extends AbstractEntityService implements Dashb
             dashboardValidator.validate(dashboard, DashboardInfo::getTenantId);
         }
         try {
+            TenantId tenantId = dashboard.getTenantId();
             if (CollectionUtils.isNotEmpty(dashboard.getResources())) {
-                resourceService.importResources(dashboard.getTenantId(), dashboard.getResources());
+                resourceService.importResources(tenantId, dashboard.getResources());
             }
             imageService.updateImagesUsage(dashboard);
-            resourceService.updateResourcesUsage(dashboard);
+            resourceService.updateResourcesUsage(tenantId, dashboard);
 
-            var saved = dashboardDao.save(dashboard.getTenantId(), dashboard);
+            var saved = dashboardDao.save(tenantId, dashboard);
             publishEvictEvent(new DashboardTitleEvictEvent(saved.getId()));
-            eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(saved.getTenantId())
+            eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId)
                     .entityId(saved.getId()).created(dashboard.getId() == null).build());
             if (dashboard.getId() == null) {
-                countService.publishCountEntityEvictEvent(saved.getTenantId(), EntityType.DASHBOARD);
+                countService.publishCountEntityEvictEvent(tenantId, EntityType.DASHBOARD);
             }
             return saved;
         } catch (Exception e) {

dao/src/main/java/org/thingsboard/server/dao/resource/BaseResourceService.java

@@ -57,6 +57,7 @@ import org.thingsboard.server.dao.service.Validator;
 import org.thingsboard.server.dao.service.validator.ResourceDataValidator;
 
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
@@ -401,18 +402,33 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
     }
 
     @Override
-    public boolean updateResourcesUsage(Dashboard dashboard) {
+    public boolean updateResourcesUsage(TenantId tenantId, Dashboard dashboard) {
+        if (dashboard.getConfiguration() == null) {
+            return false;
+        }
         Map<String, String> links = getResourcesLinks(dashboard.getResources());
-        return updateResourcesUsage(dashboard.getTenantId(), dashboard.getConfiguration(), DASHBOARD_RESOURCES_MAPPING, links);
+        return updateResourcesUsage(tenantId, List.of(dashboard.getConfiguration()), List.of(DASHBOARD_RESOURCES_MAPPING), links);
     }
 
     @Override
-    public boolean updateResourcesUsage(WidgetTypeDetails widgetTypeDetails) {
+    public boolean updateResourcesUsage(TenantId tenantId, WidgetTypeDetails widgetTypeDetails) {
         Map<String, String> links = getResourcesLinks(widgetTypeDetails.getResources());
-        boolean updated = updateResourcesUsage(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), WIDGET_RESOURCES_MAPPING, links);
+        List<JsonNode> jsonNodes = new ArrayList<>(2);
+        List<Map<String, String>> mappings = new ArrayList<>(2);
+
+        if (widgetTypeDetails.getDescriptor() != null) {
+            jsonNodes.add(widgetTypeDetails.getDescriptor());
+            mappings.add(WIDGET_RESOURCES_MAPPING);
+        }
+
         JsonNode defaultConfig = widgetTypeDetails.getDefaultConfig();
         if (defaultConfig != null) {
-            updated |= updateResourcesUsage(widgetTypeDetails.getTenantId(), defaultConfig, WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING, links);
+            jsonNodes.add(defaultConfig);
+            mappings.add(WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING);
+        }
+
+        boolean updated = updateResourcesUsage(tenantId, jsonNodes, mappings, links);
+        if (defaultConfig != null) {
             widgetTypeDetails.setDefaultConfig(defaultConfig);
         }
         return updated;
@@ -433,8 +449,9 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
         return links;
     }
 
-    private boolean updateResourcesUsage(TenantId tenantId, JsonNode jsonNode, Map<String, String> mapping, Map<String, String> links) {
-        return processResources(jsonNode, mapping, value -> {
+    private boolean updateResourcesUsage(TenantId tenantId, List<JsonNode> jsonNodes, List<Map<String, String>> mappings, Map<String, String> links) {
+        log.trace("[{}] updateResourcesUsage (new links: {}) for {}", tenantId, links, jsonNodes);
+        return processResources(jsonNodes, mappings, value -> {
             String link = getResourceLink(value);
             if (link != null) {
                 String newLink = links.get(link);
@@ -462,23 +479,31 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
     }
 
     @Override
-    public Collection<TbResourceInfo> getUsedResources(Dashboard dashboard) {
-        return getUsedResources(dashboard.getTenantId(), dashboard.getConfiguration(), DASHBOARD_RESOURCES_MAPPING).values();
+    public Collection<TbResourceInfo> getUsedResources(TenantId tenantId, Dashboard dashboard) {
+        return getUsedResources(tenantId, List.of(dashboard.getConfiguration()), List.of(DASHBOARD_RESOURCES_MAPPING)).values();
     }
 
     @Override
-    public Collection<TbResourceInfo> getUsedResources(WidgetTypeDetails widgetTypeDetails) {
-        Map<TbResourceId, TbResourceInfo> resources = getUsedResources(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor(), WIDGET_RESOURCES_MAPPING);
+    public Collection<TbResourceInfo> getUsedResources(TenantId tenantId, WidgetTypeDetails widgetTypeDetails) {
+        List<JsonNode> jsonNodes = new ArrayList<>(2);
+        List<Map<String, String>> mappings = new ArrayList<>(2);
+
+        jsonNodes.add(widgetTypeDetails.getDescriptor());
+        mappings.add(WIDGET_RESOURCES_MAPPING);
+
         JsonNode defaultConfig = widgetTypeDetails.getDefaultConfig();
         if (defaultConfig != null) {
-            resources.putAll(getUsedResources(widgetTypeDetails.getTenantId(), defaultConfig, WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING));
+            jsonNodes.add(defaultConfig);
+            mappings.add(WIDGET_DEFAULT_CONFIG_RESOURCES_MAPPING);
         }
-        return resources.values();
+
+        return getUsedResources(tenantId, jsonNodes, mappings).values();
     }
 
-    private Map<TbResourceId, TbResourceInfo> getUsedResources(TenantId tenantId, JsonNode jsonNode, Map<String, String> mapping) {
+    private Map<TbResourceId, TbResourceInfo> getUsedResources(TenantId tenantId, List<JsonNode> jsonNodes, List<Map<String, String>> mappings) {
         Map<TbResourceId, TbResourceInfo> resources = new HashMap<>();
-        processResources(jsonNode, mapping, value -> {
+        log.trace("[{}] getUsedResources for {}", tenantId, jsonNodes);
+        processResources(jsonNodes, mappings, value -> {
             String link = getResourceLink(value);
             if (link == null) {
                 return value;
@@ -517,32 +542,57 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
         }
     }
 
-    private boolean processResources(JsonNode jsonNode, Map<String, String> mapping, UnaryOperator<String> processor) {
+    private boolean processResources(List<JsonNode> jsonNodes, List<Map<String, String>> mappings, UnaryOperator<String> processor) {
         AtomicBoolean updated = new AtomicBoolean(false);
-        JacksonUtil.replaceByMapping(jsonNode, mapping, Collections.emptyMap(), (name, urlNode) -> {
-            String value = null;
-            if (urlNode.isTextual()) { // link is in the right place
-                value = urlNode.asText();
-            } else {
-                JsonNode id = urlNode.get("id"); // old structure is used
-                if (id != null && id.isTextual()) {
-                    value = id.asText();
-                }
-            }
 
-            if (StringUtils.isNotBlank(value)) {
-                value = processor.apply(value);
-            } else {
-                value = "";
-            }
+        for (int i = 0; i < jsonNodes.size(); i++) {
+            JsonNode jsonNode = jsonNodes.get(i);
+            // processing by mappings first
+            if (i <= mappings.size() - 1) {
+                JacksonUtil.replaceByMapping(jsonNode, mappings.get(i), Collections.emptyMap(), (name, urlNode) -> {
+                    String value = null;
+                    if (urlNode.isTextual()) { // link is in the right place
+                        value = urlNode.asText();
+                    } else {
+                        JsonNode id = urlNode.get("id"); // old structure is used
+                        if (id != null && id.isTextual()) {
+                            value = id.asText();
+                        }
+                    }
 
-            JsonNode newValue = new TextNode(value);
-            if (!newValue.toString().equals(urlNode.toString())) {
-                updated.set(true);
-                log.trace("Replaced '{}' with '{}'", urlNode, newValue);
+                    if (StringUtils.isNotBlank(value)) {
+                        value = processor.apply(value);
+                    } else {
+                        value = "";
+                    }
+
+                    JsonNode newValue = new TextNode(value);
+                    if (!newValue.toString().equals(urlNode.toString())) {
+                        updated.set(true);
+                        log.trace("Replaced by mapping '{}' ({}) with '{}'", value, name, newValue);
+                    }
+                    return newValue;
+                });
             }
-            return newValue;
-        });
+
+
+            // processing all
+            JacksonUtil.replaceAll(jsonNode, "", (name, value) -> {
+                if (!StringUtils.startsWith(value, DataConstants.TB_RESOURCE_PREFIX + "/api/resource/")) {
+                    return value;
+                }
+
+                String newValue = processor.apply(value);
+                if (StringUtils.equals(value, newValue)) {
+                    return value;
+                } else {
+                    updated.set(true);
+                    log.trace("Replaced '{}' ({}) with '{}'", value, name, newValue);
+                    return newValue;
+                }
+            });
+        }
+
         return updated.get();
     }
 
@@ -555,7 +605,7 @@ public class BaseResourceService extends AbstractCachedEntityService<ResourceInf
                 importResources(dashboard.getTenantId(), dashboard.getResources());
             }
             imageService.updateImagesUsage(dashboard);
-            updateResourcesUsage(dashboard);
+            updateResourcesUsage(dashboard.getTenantId(), dashboard);
 
             data = JacksonUtil.writeValueAsBytes(dashboard);
         }

dao/src/main/java/org/thingsboard/server/dao/widget/WidgetTypeServiceImpl.java

@@ -99,14 +99,15 @@ public class WidgetTypeServiceImpl implements WidgetTypeService {
         log.trace("Executing saveWidgetType [{}]", widgetTypeDetails);
         widgetTypeValidator.validate(widgetTypeDetails, WidgetType::getTenantId);
         try {
+            TenantId tenantId = widgetTypeDetails.getTenantId();
             if (CollectionUtils.isNotEmpty(widgetTypeDetails.getResources())) {
-                resourceService.importResources(widgetTypeDetails.getTenantId(), widgetTypeDetails.getResources());
+                resourceService.importResources(tenantId, widgetTypeDetails.getResources());
             }
             imageService.updateImagesUsage(widgetTypeDetails);
-            resourceService.updateResourcesUsage(widgetTypeDetails);
+            resourceService.updateResourcesUsage(tenantId, widgetTypeDetails);
 
-            WidgetTypeDetails result = widgetTypeDao.save(widgetTypeDetails.getTenantId(), widgetTypeDetails);
-            eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(result.getTenantId())
+            WidgetTypeDetails result = widgetTypeDao.save(tenantId, widgetTypeDetails);
+            eventPublisher.publishEvent(SaveEntityEvent.builder().tenantId(tenantId)
                     .entityId(result.getId()).created(widgetTypeDetails.getId() == null).build());
             return result;
         } catch (Exception t) {