Merge pull request #9753 from thingsboard/image-resources-default-images

Image gallery improvements

application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@@ -271,6 +271,12 @@ public class ThingsboardInstallService {
                         case "3.6.1":
                             log.info("Upgrading ThingsBoard from version 3.6.1 to 3.6.2 ...");
                             databaseEntitiesUpgradeService.upgradeDatabase("3.6.1");
+                            installScripts.loadSystemImages();
+                            if (!getEnv("SKIP_IMAGES_MIGRATION", false)) {
+                                installScripts.updateImages();
+                            } else {
+                                log.info("Skipping images migration. Run the upgrade with fromVersion as '3.6.2-images' to migrate");
+                            }
                             //TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache
                             break;
                         default:
@@ -324,6 +330,7 @@ public class ThingsboardInstallService {
 //                systemDataLoaderService.loadSystemPlugins();
 //                systemDataLoaderService.loadSystemRules();
                 installScripts.loadSystemLwm2mResources();
+                installScripts.loadSystemImages();
 
                 if (loadDemo) {
                     log.info("Loading demo data...");

application/src/main/java/org/thingsboard/server/service/install/InstallScripts.java

@@ -16,6 +16,7 @@
 package org.thingsboard.server.service.install;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -57,6 +58,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
+import java.util.stream.Stream;
 
 import static org.thingsboard.server.utils.LwM2mObjectModelUtils.toLwm2mResource;
 
@@ -307,6 +309,22 @@ public class InstallScripts {
         imagesUpdater.updateAssetProfilesImages();
     }
 
+    @SneakyThrows
+    public void loadSystemImages() {
+        log.info("Loading system images...");
+        Stream<Path> dashboardsFiles = Files.list(Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR));
+        try (dashboardsFiles) {
+            dashboardsFiles.forEach(file -> {
+                try {
+                    Dashboard dashboard = JacksonUtil.OBJECT_MAPPER.readValue(file.toFile(), Dashboard.class);
+                    imagesUpdater.createSystemImages(dashboard);
+                } catch (Exception e) {
+                    log.error("Failed to create system images for default dashboard {}", file.getFileName(), e);
+                }
+            });
+        }
+    }
+
     public void loadDashboards(TenantId tenantId, CustomerId customerId) throws Exception {
         Path dashboardsDir = Paths.get(getDataDir(), JSON_DIR, DEMO_DIR, DASHBOARDS_DIR);
         loadDashboardsFromDir(tenantId, customerId, dashboardsDir);

application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java

@@ -18,6 +18,7 @@ package org.thingsboard.server.service.install.update;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Component;
+import org.thingsboard.server.common.data.Dashboard;
 import org.thingsboard.server.common.data.HasImage;
 import org.thingsboard.server.common.data.id.EntityId;
 import org.thingsboard.server.common.data.id.TenantId;
@@ -62,15 +63,23 @@ public class ImagesUpdater {
         updateImages(dashboardsIds, "dashboard", imageService::replaceBase64WithImageUrl, dashboardDao);
     }
 
+    public void createSystemImages(Dashboard defaultDashboard) {
+        defaultDashboard.setTenantId(TenantId.SYS_TENANT_ID);
+        boolean created = imageService.replaceBase64WithImageUrl(defaultDashboard);
+        if (created) {
+            log.debug("Created system images for default dashboard '{}'", defaultDashboard.getTitle());
+        }
+    }
+
     public void updateDeviceProfilesImages() {
         log.info("Updating device profiles images...");
-        var deviceProfiles = new PageDataIterable<>(deviceProfileDao::findAll, 256);
+        var deviceProfiles = new PageDataIterable<>(deviceProfileDao::findAllWithImages, 256);
         updateImages(deviceProfiles, "device profile", imageService::replaceBase64WithImageUrl, deviceProfileDao);
     }
 
     public void updateAssetProfilesImages() {
         log.info("Updating asset profiles images...");
-        var assetProfiles = new PageDataIterable<>(assetProfileDao::findAll, 256);
+        var assetProfiles = new PageDataIterable<>(assetProfileDao::findAllWithImages, 256);
         updateImages(assetProfiles, "asset profile", imageService::replaceBase64WithImageUrl, assetProfileDao);
     }
 

application/src/main/java/org/thingsboard/server/service/resource/DefaultTbResourceService.java

@@ -53,6 +53,9 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
 
     @Override
     public TbResource save(TbResource resource, User user) throws ThingsboardException {
+        if (resource.getResourceType() == ResourceType.IMAGE) {
+            throw new IllegalArgumentException("Image resource type is not supported");
+        }
         ActionType actionType = resource.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
         TenantId tenantId = resource.getTenantId();
         try {
@@ -74,6 +77,9 @@ public class DefaultTbResourceService extends AbstractTbEntityService implements
 
     @Override
     public void delete(TbResource tbResource, User user) {
+        if (tbResource.getResourceType() == ResourceType.IMAGE) {
+            throw new IllegalArgumentException("Image resource type is not supported");
+        }
         TbResourceId resourceId = tbResource.getId();
         TenantId tenantId = tbResource.getTenantId();
         try {

dao/src/main/java/org/thingsboard/server/dao/asset/AssetProfileDao.java

@@ -45,6 +45,6 @@ public interface AssetProfileDao extends Dao<AssetProfile>, ExportableEntityDao<
 
     AssetProfile findByName(TenantId tenantId, String profileName);
 
-    PageData<AssetProfile> findAll(PageLink pageLink);
+    PageData<AssetProfile> findAllWithImages(PageLink pageLink);
 
 }

dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileDao.java

@@ -47,6 +47,6 @@ public interface DeviceProfileDao extends Dao<DeviceProfile>, ExportableEntityDa
 
     DeviceProfile findByName(TenantId tenantId, String profileName);
 
-    PageData<DeviceProfile> findAll(PageLink pageLink);
+    PageData<DeviceProfile> findAllWithImages(PageLink pageLink);
 
 }

dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java

@@ -24,6 +24,7 @@ import lombok.Data;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Propagation;
@@ -148,6 +149,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
         image.setDescriptorValue(descriptor);
         image.setPreview(result.getRight());
 
+        log.debug("[{}] Creating image {} ('{}')", image.getTenantId(), image.getResourceKey(), image.getName());
         return new TbResourceInfo(doSaveResource(image));
     }
 
@@ -188,21 +190,25 @@ public class BaseImageService extends BaseResourceService implements ImageServic
             resourceKey = basename + "_(" + idx + ")." + extension;
             idx++;
         }
+        log.debug("[{}] Generated unique key {} for image {}", tenantId, resourceKey, filename);
         return resourceKey;
     }
 
     @Override
     public TbResourceInfo saveImageInfo(TbResourceInfo imageInfo) {
+        log.trace("Executing saveImageInfo [{}] [{}]", imageInfo.getTenantId(), imageInfo.getId());
         return saveResource(new TbResource(imageInfo));
     }
 
     @Override
     public TbResourceInfo getImageInfoByTenantIdAndKey(TenantId tenantId, String key) {
+        log.trace("Executing getImageInfoByTenantIdAndKey [{}] [{}]", tenantId, key);
         return findResourceInfoByTenantIdAndKey(tenantId, ResourceType.IMAGE, key);
     }
 
     @Override
     public PageData<TbResourceInfo> getImagesByTenantId(TenantId tenantId, PageLink pageLink) {
+        log.trace("Executing getImagesByTenantId [{}]", tenantId);
         TbResourceInfoFilter filter = TbResourceInfoFilter.builder()
                 .tenantId(tenantId)
                 .resourceTypes(Set.of(ResourceType.IMAGE))
@@ -212,6 +218,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
 
     @Override
     public PageData<TbResourceInfo> getAllImagesByTenantId(TenantId tenantId, PageLink pageLink) {
+        log.trace("Executing getAllImagesByTenantId [{}]", tenantId);
         TbResourceInfoFilter filter = TbResourceInfoFilter.builder()
                 .tenantId(tenantId)
                 .resourceTypes(Set.of(ResourceType.IMAGE))
@@ -221,11 +228,13 @@ public class BaseImageService extends BaseResourceService implements ImageServic
 
     @Override
     public byte[] getImageData(TenantId tenantId, TbResourceId imageId) {
+        log.trace("Executing getImageData [{}] [{}]", tenantId, imageId);
         return resourceDao.getResourceData(tenantId, imageId);
     }
 
     @Override
     public byte[] getImagePreview(TenantId tenantId, TbResourceId imageId) {
+        log.trace("Executing getImagePreview [{}] [{}]", tenantId, imageId);
         return resourceDao.getResourcePreview(tenantId, imageId);
     }
 
@@ -260,12 +269,14 @@ public class BaseImageService extends BaseResourceService implements ImageServic
 
     @Override
     public TbResourceInfo findSystemOrTenantImageByEtag(TenantId tenantId, String etag) {
+        log.trace("Executing findSystemOrTenantImageByEtag [{}] [{}]", tenantId, etag);
         return resourceInfoDao.findSystemOrTenantImageByEtag(tenantId, ResourceType.IMAGE, etag);
     }
 
     @Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure
     @Override
     public boolean replaceBase64WithImageUrl(HasImage entity, String type) {
+        log.trace("Executing replaceBase64WithImageUrl [{}] [{}] [{}]", entity.getTenantId(), type, entity.getName());
         String imageName = "\"" + entity.getName() + "\" ";
         if (entity.getTenantId() == null || entity.getTenantId().isSysTenantId()) {
             imageName += "system ";
@@ -280,6 +291,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
     @Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure
     @Override
     public boolean replaceBase64WithImageUrl(WidgetTypeDetails entity) {
+        log.trace("Executing replaceBase64WithImageUrl [{}] [WidgetTypeDetails] [{}]", entity.getTenantId(), entity.getId());
         String prefix = "\"" + entity.getName() + "\" ";
         if (entity.getTenantId() == null || entity.getTenantId().isSysTenantId()) {
             prefix += "system ";
@@ -305,6 +317,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
     @Transactional(propagation = Propagation.NOT_SUPPORTED)// we don't want transaction to rollback in case of an image processing failure
     @Override
     public boolean replaceBase64WithImageUrl(Dashboard entity) {
+        log.trace("Executing replaceBase64WithImageUrl [{}] [Dashboard] [{}]", entity.getTenantId(), entity.getId());
         String prefix = "\"" + entity.getTitle() + "\" dashboard";
         var result = base64ToImageUrl(entity.getTenantId(), prefix + " image", entity.getImage());
         boolean updated = result.isUpdated();
@@ -442,7 +455,11 @@ public class BaseImageService extends BaseResourceService implements ImageServic
             try {
                 imageInfo = saveImage(image);
             } catch (Exception e) {
-                log.warn("[{}][{}] Failed to replace Base64 with image url: {}", tenantId, name, StringUtils.abbreviate(data, 50), e);
+                if (log.isDebugEnabled()) { // printing stacktrace
+                    log.warn("[{}][{}] Failed to replace Base64 with image url for {}", tenantId, name, StringUtils.abbreviate(data, 50), e);
+                } else {
+                    log.warn("[{}][{}] Failed to replace Base64 with image url for {}: {}", tenantId, name, StringUtils.abbreviate(data, 50), ExceptionUtils.getMessage(e));
+                }
                 return UpdateResult.of(false, data);
             }
         }
@@ -492,17 +509,20 @@ public class BaseImageService extends BaseResourceService implements ImageServic
 
     @Override
     public void inlineImage(HasImage entity) {
+        log.trace("Executing inlineImage [{}] [{}] [{}]", entity.getTenantId(), entity.getClass().getSimpleName(), entity.getName());
         entity.setImage(inlineImage(entity.getTenantId(), "image", entity.getImage()));
     }
 
     @Override
     public void inlineImages(Dashboard dashboard) {
+        log.trace("Executing inlineImage [{}] [Dashboard] [{}]", dashboard.getTenantId(), dashboard.getId());
         inlineImage(dashboard);
         inlineIntoJson(dashboard.getTenantId(), dashboard.getConfiguration());
     }
 
     @Override
     public void inlineImages(WidgetTypeDetails widgetTypeDetails) {
+        log.trace("Executing inlineImage [{}] [WidgetTypeDetails] [{}]", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId());
         inlineImage(widgetTypeDetails);
         inlineIntoJson(widgetTypeDetails.getTenantId(), widgetTypeDetails.getDescriptor());
     }

dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceDao.java

@@ -46,4 +46,6 @@ public interface TbResourceDao extends Dao<TbResource>, TenantEntityWithDataDao,
 
     byte[] getResourcePreview(TenantId tenantId, TbResourceId resourceId);
 
+    long getResourceSize(TenantId tenantId, TbResourceId resourceId);
+
 }

dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java

@@ -161,7 +161,7 @@ public abstract class DataValidator<D extends BaseData<?>> {
     }
 
     static void validateQueueNameOrTopic(String value, String fieldName) {
-        if (StringUtils.isEmpty(value) || value.trim().length() == 0 ) {
+        if (StringUtils.isEmpty(value) || value.trim().length() == 0) {
             throw new DataValidationException(String.format("Queue %s should be specified!", fieldName));
         }
         if (!QUEUE_PATTERN.matcher(value).matches()) {

dao/src/main/java/org/thingsboard/server/dao/service/validator/ResourceDataValidator.java

@@ -90,7 +90,12 @@ public class ResourceDataValidator extends DataValidator<TbResource> {
                 throw new IllegalArgumentException("Resource exceeds the maximum size of " + maxResourceSize + " bytes");
             }
             long maxSumResourcesDataInBytes = profileConfiguration.getMaxResourcesInBytes();
-            validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, resource.getData().length, TB_RESOURCE);
+            int dataSize = resource.getData().length;
+            if (resource.getId() != null) {
+                long prevSize = resourceDao.getResourceSize(tenantId, resource.getId());
+                dataSize -= prevSize;
+            }
+            validateMaxSumDataSizePerTenant(tenantId, resourceDao, maxSumResourcesDataInBytes, dataSize, TB_RESOURCE);
         }
         if (StringUtils.isEmpty(resource.getFileName())) {
             throw new DataValidationException("Resource file name should be specified!");

dao/src/main/java/org/thingsboard/server/dao/sql/asset/AssetProfileRepository.java

@@ -23,7 +23,6 @@ import org.springframework.data.repository.query.Param;
 import org.thingsboard.server.common.data.asset.AssetProfileInfo;
 import org.thingsboard.server.dao.ExportableEntityRepository;
 import org.thingsboard.server.dao.model.sql.AssetProfileEntity;
-import org.thingsboard.server.dao.model.sql.WidgetsBundleEntity;
 
 import java.util.List;
 import java.util.UUID;
@@ -70,4 +69,6 @@ public interface AssetProfileRepository extends JpaRepository<AssetProfileEntity
             "FROM AssetProfileEntity a WHERE a.image = :imageLink")
     List<AssetProfileInfo> findByImageLink(@Param("imageLink") String imageLink, Pageable page);
 
+    Page<AssetProfileEntity> findAllByImageNotNull(Pageable pageable);
+
 }

dao/src/main/java/org/thingsboard/server/dao/sql/asset/JpaAssetProfileDao.java

@@ -99,8 +99,8 @@ public class JpaAssetProfileDao extends JpaAbstractDao<AssetProfileEntity, Asset
     }
 
     @Override
-    public PageData<AssetProfile> findAll(PageLink pageLink) {
-        return DaoUtil.toPageData(assetProfileRepository.findAll(DaoUtil.toPageable(pageLink)));
+    public PageData<AssetProfile> findAllWithImages(PageLink pageLink) {
+        return DaoUtil.toPageData(assetProfileRepository.findAllByImageNotNull(DaoUtil.toPageable(pageLink)));
     }
 
     @Override

dao/src/main/java/org/thingsboard/server/dao/sql/device/DeviceProfileRepository.java

@@ -81,4 +81,6 @@ public interface DeviceProfileRepository extends JpaRepository<DeviceProfileEnti
     @Query("SELECT externalId FROM DeviceProfileEntity WHERE id = :id")
     UUID getExternalIdById(@Param("id") UUID id);
 
+    Page<DeviceProfileEntity> findAllByImageNotNull(Pageable pageable);
+
 }

dao/src/main/java/org/thingsboard/server/dao/sql/device/JpaDeviceProfileDao.java

@@ -117,8 +117,8 @@ public class JpaDeviceProfileDao extends JpaAbstractDao<DeviceProfileEntity, Dev
     }
 
     @Override
-    public PageData<DeviceProfile> findAll(PageLink pageLink) {
-        return DaoUtil.toPageData(deviceProfileRepository.findAll(DaoUtil.toPageable(pageLink)));
+    public PageData<DeviceProfile> findAllWithImages(PageLink pageLink) {
+        return DaoUtil.toPageData(deviceProfileRepository.findAllByImageNotNull(DaoUtil.toPageable(pageLink)));
     }
 
     @Override

dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceDao.java

@@ -104,6 +104,11 @@ public class JpaTbResourceDao extends JpaAbstractDao<TbResourceEntity, TbResourc
         return resourceRepository.getPreviewById(resourceId.getId());
     }
 
+    @Override
+    public long getResourceSize(TenantId tenantId, TbResourceId resourceId) {
+        return resourceRepository.getDataSizeById(resourceId.getId());
+    }
+
     @Override
     public Long sumDataSizeByTenantId(TenantId tenantId) {
         return resourceRepository.sumDataSizeByTenantId(tenantId.getId());

dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceInfoDao.java

@@ -107,6 +107,6 @@ public class JpaTbResourceInfoDao extends JpaAbstractDao<TbResourceInfoEntity, T
 
     @Override
     public TbResourceInfo findSystemOrTenantImageByEtag(TenantId tenantId, ResourceType resourceType, String etag) {
-        return DaoUtil.getData(resourceInfoRepository.findSystemOrTenantImageByEtag(TenantId.SYS_TENANT_ID.getId(), tenantId.getId(), resourceType.name(), etag));
+        return DaoUtil.getData(resourceInfoRepository.findSystemOrTenantImageByEtag(tenantId.getId(), resourceType.name(), etag));
     }
 }

dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceInfoRepository.java

@@ -65,9 +65,9 @@ public interface TbResourceInfoRepository extends JpaRepository<TbResourceInfoEn
 
     List<TbResourceInfoEntity> findByTenantIdAndEtagAndResourceKeyStartingWith(UUID tenantId, String etag, String query);
 
-    @Query(value = "SELECT * FROM resource r WHERE (r.tenant_id = :systemTenantId OR r.tenant_id = :tenantId) AND r.resource_type = :resourceType AND r.etag = :etag LIMIT 1", nativeQuery = true)
-    TbResourceInfoEntity findSystemOrTenantImageByEtag(@Param("systemTenantId") UUID sysTenantId,
-                                                       @Param("tenantId") UUID tenantId,
+    @Query(value = "SELECT * FROM resource r WHERE (r.tenant_id = '13814000-1dd2-11b2-8080-808080808080' OR r.tenant_id = :tenantId) " +
+            "AND r.resource_type = :resourceType AND r.etag = :etag LIMIT 1", nativeQuery = true)
+    TbResourceInfoEntity findSystemOrTenantImageByEtag(@Param("tenantId") UUID tenantId,
                                                        @Param("resourceType") String resourceType,
                                                        @Param("etag") String etag);
 }

dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceRepository.java

@@ -88,6 +88,9 @@ public interface TbResourceRepository extends JpaRepository<TbResourceEntity, UU
     @Query(value = "SELECT COALESCE(preview, data) FROM resource WHERE id = :id", nativeQuery = true)
     byte[] getPreviewById(@Param("id") UUID id);
 
+    @Query(value = "SELECT length(r.data) FROM resource r WHERE r.id = :id", nativeQuery = true)
+    long getDataSizeById(@Param("id") UUID id);
+
     @Query("SELECT externalId FROM TbResourceInfoEntity WHERE id = :id")
     UUID getExternalIdByInternal(@Param("id") UUID internalId);
 

ui-ngx/src/app/shared/models/resource.models.ts

@@ -24,8 +24,7 @@ export enum ResourceType {
   LWM2M_MODEL = 'LWM2M_MODEL',
   PKCS_12 = 'PKCS_12',
   JKS = 'JKS',
-  JS_MODULE = 'JS_MODULE',
-  IMAGE = 'IMAGE'
+  JS_MODULE = 'JS_MODULE'
 }
 
 export const ResourceTypeMIMETypes = new Map<ResourceType, string>(
@@ -33,8 +32,7 @@ export const ResourceTypeMIMETypes = new Map<ResourceType, string>(
     [ResourceType.LWM2M_MODEL, 'application/xml,text/xml'],
     [ResourceType.PKCS_12, 'application/x-pkcs12'],
     [ResourceType.JKS, 'application/x-java-keystore'],
-    [ResourceType.JS_MODULE, 'text/javascript,application/javascript'],
-    [ResourceType.IMAGE, 'image/*']
+    [ResourceType.JS_MODULE, 'text/javascript,application/javascript']
   ]
 );
 
@@ -52,8 +50,7 @@ export const ResourceTypeTranslationMap = new Map<ResourceType, string>(
     [ResourceType.LWM2M_MODEL, 'resource.type.lwm2m-model'],
     [ResourceType.PKCS_12, 'resource.type.pkcs-12'],
     [ResourceType.JKS, 'resource.type.jks'],
-    [ResourceType.JS_MODULE, 'resource.type.js-module'],
-    [ResourceType.IMAGE, 'resource.type.image']
+    [ResourceType.JS_MODULE, 'resource.type.js-module']
   ]
 );
 

ui-ngx/src/assets/locale/locale.constant-en_US.json

@@ -3692,8 +3692,7 @@
             "jks": "JKS",
             "js-module": "JS module",
             "lwm2m-model": "LWM2M model",
-            "pkcs-12": "PKCS #12",
-            "image": "Image"
+            "pkcs-12": "PKCS #12"
         }
     },
     "rulechain": {