tsai
/
thingsboard
mirror of https://github.com/thingsboard/thingsboard.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Fixes and improvements for images processing

pull/9542/head
ViacheslavKlimov 3 years ago
parent
ddfa55ba6e
commit
2d26fdcb95
8 changed files with 132 additions and 102 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 117
      application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java
  2. 4
      dao/pom.xml
  3. 25
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
  4. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java
  5. 2
      dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceRepository.java
  6. 8
      dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java
  7. 70
      dao/src/main/java/org/thingsboard/server/dao/util/ImageUtils.java
  8. 6
      pom.xml

117
application/src/main/java/org/thingsboard/server/service/install/update/ImagesUpdater.java
View File

@ -18,15 +18,11 @@ package org.thingsboard.server.service.install.update;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.Dashboard;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.asset.AssetProfile;
import org.thingsboard.server.common.data.id.DashboardId;
import org.thingsboard.server.common.data.HasImage;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.id.WidgetTypeId;
import org.thingsboard.server.common.data.page.PageDataIterable;
import org.thingsboard.server.common.data.widget.WidgetTypeDetails;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.Dao;
import org.thingsboard.server.dao.asset.AssetProfileDao;
import org.thingsboard.server.dao.dashboard.DashboardDao;
import org.thingsboard.server.dao.device.DeviceProfileDao;
@ -34,6 +30,9 @@ import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.widget.WidgetTypeDao;
import org.thingsboard.server.dao.widget.WidgetsBundleDao;
import java.util.function.BiFunction;
import java.util.function.Function;
@Component
@RequiredArgsConstructor
@Slf4j
@ -48,108 +47,78 @@ public class ImagesUpdater {
public void updateWidgetsBundlesImages() {
log.info("Updating widgets bundles images...");
var widgetsBundles = new PageDataIterable<>(widgetsBundleDao::findAllWidgetsBundles, 128);
int updatedCount = 0;
int totalCount = 0;
for (WidgetsBundle widgetsBundle : widgetsBundles) {
totalCount++;
try {
boolean updated = imageService.replaceBase64WithImageUrl(widgetsBundle, "bundle");
if (updated) {
widgetsBundleDao.save(widgetsBundle.getTenantId(), widgetsBundle);
log.debug("[{}][{}][{}] Updated widgets bundle images", widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle.getTitle());
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update widgets bundle images", widgetsBundle.getTenantId(), widgetsBundle.getId(), widgetsBundle.getTitle(), e);
}
}
log.info("Updated {} widgets bundles out of {}", updatedCount, totalCount);
updateImages(widgetsBundles, "bundle", imageService::replaceBase64WithImageUrl, widgetsBundleDao);
}
public void updateWidgetTypesImages() {
log.info("Updating widget types images...");
var widgetTypes = new PageDataIterable<>(widgetTypeDao::findAllWidgetTypesIds, 1024);
int updatedCount = 0;
int totalCount = 0;
for (WidgetTypeId widgetTypeId : widgetTypes) {
totalCount++;
WidgetTypeDetails widgetTypeDetails = widgetTypeDao.findById(TenantId.SYS_TENANT_ID, widgetTypeId.getId());
try {
boolean updated = imageService.replaceBase64WithImageUrl(widgetTypeDetails);
if (updated) {
widgetTypeDao.save(widgetTypeDetails.getTenantId(), widgetTypeDetails);
log.debug("[{}][{}][{}] Updated widget type images", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId(), widgetTypeDetails.getName());
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update widget type images", widgetTypeDetails.getTenantId(), widgetTypeDetails.getId(), widgetTypeDetails.getName(), e);
}
}
log.info("Updated {} widget types out of {}", updatedCount, totalCount);
var widgetTypesIds = new PageDataIterable<>(widgetTypeDao::findAllWidgetTypesIds, 1024);
updateImages(widgetTypesIds, "widget type", imageService::replaceBase64WithImageUrl, widgetTypeDao);
}
public void updateDashboardsImages() {
log.info("Updating dashboards images...");
var dashboards = new PageDataIterable<>(dashboardDao::findAllIds, 1024);
int updatedCount = 0;
int totalCount = 0;
for (DashboardId dashboardId : dashboards) {
totalCount++;
Dashboard dashboard = dashboardDao.findById(TenantId.SYS_TENANT_ID, dashboardId.getId());
try {
boolean updated = imageService.replaceBase64WithImageUrl(dashboard);
if (updated) {
dashboardDao.save(dashboard.getTenantId(), dashboard);
log.info("[{}][{}][{}] Updated dashboard images", dashboard.getTenantId(), dashboardId, dashboard.getTitle());
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update dashboard images", dashboard.getTenantId(), dashboardId, dashboard.getTitle(), e);
}
}
log.info("Updated {} dashboards out of {}", updatedCount, totalCount);
var dashboardsIds = new PageDataIterable<>(dashboardDao::findAllIds, 1024);
updateImages(dashboardsIds, "dashboard", imageService::replaceBase64WithImageUrl, dashboardDao);
}
public void updateDeviceProfilesImages() {
log.info("Updating device profiles images...");
var deviceProfiles = new PageDataIterable<>(deviceProfileDao::findAll, 256);
updateImages(deviceProfiles, "device profile", imageService::replaceBase64WithImageUrl, deviceProfileDao);
}
public void updateAssetProfilesImages() {
log.info("Updating asset profiles images...");
var assetProfiles = new PageDataIterable<>(assetProfileDao::findAll, 256);
updateImages(assetProfiles, "asset profile", imageService::replaceBase64WithImageUrl, assetProfileDao);
}
private <E extends HasImage> void updateImages(Iterable<E> entities, String type,
BiFunction<E, String, Boolean> updater, Dao<E> dao) {
int updatedCount = 0;
int totalCount = 0;
for (DeviceProfile deviceProfile : deviceProfiles) {
for (E entity : entities) {
totalCount++;
try {
boolean updated = imageService.replaceBase64WithImageUrl(deviceProfile, "device profile");
boolean updated = updater.apply(entity, type);
if (updated) {
deviceProfileDao.save(deviceProfile.getTenantId(), deviceProfile);
log.debug("[{}][{}][{}] Updated device profile images", deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile.getName());
dao.save(entity.getTenantId(), entity);
log.debug("[{}][{}] Updated {} images", entity.getTenantId(), entity.getName(), type);
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update device profile images", deviceProfile.getTenantId(), deviceProfile.getId(), deviceProfile.getName(), e);
log.error("[{}][{}] Failed to update {} images", entity.getTenantId(), entity.getName(), type, e);
}
if (totalCount % 100 == 0) {
log.info("Processed {} {}s so far", totalCount, type);
}
}
log.info("Updated {} device profiles out of {}", updatedCount, totalCount);
log.info("Updated {} {}s out of {}", updatedCount, type, totalCount);
}
public void updateAssetProfilesImages() {
log.info("Updating asset profiles images...");
var assetProfiles = new PageDataIterable<>(assetProfileDao::findAll, 256);
private <E extends HasImage> void updateImages(Iterable<? extends EntityId> entitiesIds, String type,
Function<E, Boolean> updater, Dao<E> dao) {
int updatedCount = 0;
int totalCount = 0;
for (AssetProfile assetProfile : assetProfiles) {
for (EntityId id : entitiesIds) {
totalCount++;
E entity = dao.findById(TenantId.SYS_TENANT_ID, id.getId());
try {
boolean updated = imageService.replaceBase64WithImageUrl(assetProfile, "asset profile");
boolean updated = updater.apply(entity);
if (updated) {
assetProfileDao.save(assetProfile.getTenantId(), assetProfile);
log.debug("[{}][{}][{}] Updated asset profile images", assetProfile.getTenantId(), assetProfile.getId(), assetProfile.getName());
dao.save(entity.getTenantId(), entity);
log.debug("[{}][{}] Updated {} images", entity.getTenantId(), entity.getName(), type);
updatedCount++;
}
} catch (Exception e) {
log.error("[{}][{}][{}] Failed to update asset profile images", assetProfile.getTenantId(), assetProfile.getId(), assetProfile.getName(), e);
log.error("[{}][{}] Failed to update {} images", entity.getTenantId(), entity.getName(), type, e);
}
if (totalCount % 100 == 0) {
log.info("Processed {} {}s so far", totalCount, type);
}
}
log.info("Updated {} asset profiles out of {}", updatedCount, totalCount);
log.info("Updated {} {}s out of {}", updatedCount, type, totalCount);
}
}

4
dao/pom.xml
View File

@ -242,6 +242,10 @@
<groupId>org.apache.xmlgraphics</groupId>
<artifactId>batik-codec</artifactId>
</dependency>
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
</dependency>
</dependencies>
<build>
<plugins>

25
dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
View File

@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.base.Strings;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
@ -285,8 +286,10 @@ public class BaseImageService extends BaseResourceService implements ImageServic
if (entity.getDescriptor().isObject()) {
ObjectNode descriptor = (ObjectNode) entity.getDescriptor();
JsonNode defaultConfig = JacksonUtil.toJsonNode(descriptor.get("defaultConfig").asText());
updated |= base64ToImageUrlUsingMapping(entity.getTenantId(), WIDGET_TYPE_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), defaultConfig);
descriptor.put("defaultConfig", defaultConfig.toString());
if (defaultConfig != null && defaultConfig.isObject()) {
updated |= base64ToImageUrlUsingMapping(entity.getTenantId(), WIDGET_TYPE_BASE64_MAPPING, Collections.singletonMap("prefix", prefix), defaultConfig);
descriptor.put("defaultConfig", defaultConfig.toString());
}
}
updated |= base64ToImageUrlRecursively(entity.getTenantId(), prefix, entity.getDescriptor());
return updated;
@ -313,6 +316,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic
JsonPathProcessingTask task = tasks.poll();
String token = task.currentToken();
JsonNode node = task.getNode();
if (node == null) {
continue;
}
if (token.equals("*") || token.startsWith("$")) {
String variableName = token.startsWith("$") ? token.substring(1) : null;
if (node.isArray()) {
@ -345,8 +351,8 @@ public class BaseImageService extends BaseResourceService implements ImageServic
}
if (task.isLast()) {
String name = expression;
for (var replacements : task.getVariables().entrySet()) {
name = name.replace("$" + replacements.getKey(), replacements.getValue());
for (var replacement : task.getVariables().entrySet()) {
name = name.replace("$" + replacement.getKey(), Strings.nullToEmpty(replacement.getValue()));
}
if (node.isObject() && value.isTextual()) {
var result = base64ToImageUrl(tenantId, name, value.asText());
@ -362,7 +368,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
}
}
} else {
if (StringUtils.isNotEmpty(variableName) && StringUtils.isNotEmpty(variableValue)) {
if (StringUtils.isNotEmpty(variableName)) {
tasks.add(task.next(value, variableName, variableValue));
} else {
tasks.add(task.next(value));
@ -404,6 +410,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic
byte[] imageData = Base64.getDecoder().decode(base64Data);
String etag = calculateEtag(imageData);
var imageInfo = findImageByTenantIdAndEtag(tenantId, etag);
if (imageInfo == null && !tenantId.isSysTenantId()) {
imageInfo = findImageByTenantIdAndEtag(TenantId.SYS_TENANT_ID, etag);
}
if (imageInfo == null) {
TbResource image = new TbResource();
image.setTenantId(tenantId);
@ -415,9 +424,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic
String fileName;
if (StringUtils.isBlank(mdResourceKey)) {
fileName = mdResourceName.toLowerCase()
.replace("'", "").replace("\"", "")
.replace(" ", "_").replace("/", "_")
fileName = StringUtils.strip(mdResourceName.toLowerCase()
.replaceAll("['\"]", "")
.replaceAll("[^\\pL\\d]+", "_"), "_") // leaving only letters and numbers
+ "." + extension;
} else {
fileName = mdResourceKey;

2
dao/src/main/java/org/thingsboard/server/dao/sql/dashboard/DashboardRepository.java
View File

@ -43,7 +43,7 @@ public interface DashboardRepository extends JpaRepository<DashboardEntity, UUID
@Query("SELECT d.id FROM DashboardEntity d WHERE d.tenantId = :tenantId")
Page<UUID> findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable);
@Query("SELECT id FROM DashboardEntity")
@Query("SELECT d.id FROM DashboardEntity d")
Page<UUID> findAllIds(Pageable pageable);
}

2
dao/src/main/java/org/thingsboard/server/dao/sql/resource/TbResourceRepository.java
View File

@ -91,7 +91,7 @@ public interface TbResourceRepository extends JpaRepository<TbResourceEntity, UU
@Query("SELECT externalId FROM TbResourceInfoEntity WHERE id = :id")
UUID getExternalIdByInternal(@Param("id") UUID internalId);
@Query("SELECT id FROM TbResourceInfoEntity WHERE tenantId = :tenantId")
@Query("SELECT r.id FROM TbResourceInfoEntity r WHERE r.tenantId = :tenantId")
Page<UUID> findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable);
}

8
dao/src/main/java/org/thingsboard/server/dao/sql/widget/WidgetTypeRepository.java
View File

@ -69,17 +69,17 @@ public interface WidgetTypeRepository extends JpaRepository<WidgetTypeDetailsEnt
@Query(value = "SELECT * FROM widget_type wt " +
"WHERE wt.tenant_id = :tenantId AND cast(wt.descriptor as json) ->> 'resources' LIKE LOWER(CONCAT('%', :resourceId, '%'))",
nativeQuery = true)
nativeQuery = true)
List<WidgetTypeDetailsEntity> findWidgetTypesInfosByTenantIdAndResourceId(@Param("tenantId") UUID tenantId,
@Param("resourceId") UUID resourceId);
@Param("resourceId") UUID resourceId);
@Query("SELECT externalId FROM WidgetTypeDetailsEntity WHERE id = :id")
UUID getExternalIdById(@Param("id") UUID id);
@Query("SELECT id FROM WidgetTypeDetailsEntity")
@Query("SELECT w.id FROM WidgetTypeDetailsEntity w")
Page<UUID> findAllIds(Pageable pageable);
@Query("SELECT id FROM WidgetTypeDetailsEntity WHERE tenantId = :tenantId")
@Query("SELECT w.id FROM WidgetTypeDetailsEntity w WHERE w.tenantId = :tenantId")
Page<UUID> findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable);
}

70
dao/src/main/java/org/thingsboard/server/dao/util/ImageUtils.java
View File

@ -15,9 +15,14 @@
*/
package org.thingsboard.server.dao.util;
import com.drew.imaging.ImageMetadataReader;
import com.drew.metadata.Directory;
import com.drew.metadata.Metadata;
import com.drew.metadata.Tag;
import lombok.AccessLevel;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.batik.anim.dom.SAXSVGDocumentFactory;
import org.apache.batik.bridge.BridgeContext;
import org.apache.batik.bridge.DocumentLoader;
@ -41,11 +46,13 @@ import java.io.ByteArrayOutputStream;
import java.util.Map;
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@Slf4j
public class ImageUtils {
private static final Map<String, String> mediaTypeMappings = Map.of(
"jpeg", "jpg",
"svg+xml", "svg"
"svg+xml", "svg",
"x-icon", "ico"
);
public static String mediaTypeToFileExtension(String mimeType) {
@ -64,15 +71,49 @@ public class ImageUtils {
if (mediaTypeToFileExtension(mediaType).equals("svg")) {
return processSvgImage(data, mediaType, thumbnailMaxDimension);
}
BufferedImage bufferedImage = ImageIO.read(new ByteArrayInputStream(data));
ProcessedImage image = new ProcessedImage();
image.setMediaType(mediaType);
image.setWidth(bufferedImage.getWidth());
image.setHeight(bufferedImage.getHeight());
image.setData(data);
image.setSize(data.length);
BufferedImage bufferedImage = null;
try {
bufferedImage = ImageIO.read(new ByteArrayInputStream(data));
} catch (Exception ignored) {
}
if (bufferedImage == null) { // means that media type is not supported by ImageIO; extracting width and height from metadata and leaving preview as original image
Metadata metadata = ImageMetadataReader.readMetadata(new ByteArrayInputStream(data));
for (Directory dir : metadata.getDirectories()) {
Tag widthTag = dir.getTags().stream()
.filter(tag -> tag.getTagName().toLowerCase().contains("width"))
.findFirst().orElse(null);
Tag heightTag = dir.getTags().stream()
.filter(tag -> tag.getTagName().toLowerCase().contains("height"))
.findFirst().orElse(null);
if (widthTag == null || heightTag == null) {
continue;
}
int width = Integer.parseInt(dir.getObject(widthTag.getTagType()).toString());
int height = Integer.parseInt(dir.getObject(widthTag.getTagType()).toString());
image.setWidth(width);
image.setHeight(height);
ProcessedImage preview = new ProcessedImage();
preview.setWidth(image.getWidth());
preview.setHeight(image.getHeight());
preview.setMediaType(mediaType);
preview.setData(null);
preview.setSize(data.length);
image.setPreview(preview);
log.debug("Couldn't parse {} ({}) with ImageIO, got width {} and height {} from metadata", mediaType, dir.getName(), width, height);
return image;
}
throw new IllegalArgumentException("Media type " + mediaType + " not supported");
}
image.setWidth(bufferedImage.getWidth());
image.setHeight(bufferedImage.getHeight());
ProcessedImage preview = new ProcessedImage();
int[] thumbnailDimensions = getThumbnailDimensions(image.getWidth(), image.getHeight(), thumbnailMaxDimension);
preview.setWidth(thumbnailDimensions[0]);
@ -101,18 +142,19 @@ public class ImageUtils {
}
public static ProcessedImage processSvgImage(byte[] data, String mediaType, int thumbnailMaxDimension) throws Exception {
SAXSVGDocumentFactory factory = new SAXSVGDocumentFactory(
XMLResourceDescriptor.getXMLParserClassName());
Document document = factory.createDocument(
null, new ByteArrayInputStream(data));
SAXSVGDocumentFactory factory = new SAXSVGDocumentFactory(XMLResourceDescriptor.getXMLParserClassName());
Document document = factory.createDocument(null, new ByteArrayInputStream(data));
Integer width = null;
Integer height = null;
String strWidth = document.getDocumentElement().getAttribute("width");
String strHeight = document.getDocumentElement().getAttribute("height");
if (StringUtils.isNotEmpty(strWidth) && StringUtils.isNotEmpty(strHeight)) {
width = (int) Double.parseDouble(strWidth);
height = (int) Double.parseDouble(strHeight);;
} else {
try {
width = (int) Double.parseDouble(strWidth);
height = (int) Double.parseDouble(strHeight);
} catch (NumberFormatException ignored) {} // in case width and height are in %, mm, etc.
}
if (width == null || height == null) {
String viewBox = document.getDocumentElement().getAttribute("viewBox");
if (StringUtils.isNotEmpty(viewBox)) {
String[] viewBoxValues = viewBox.split(" ");
@ -124,10 +166,10 @@ public class ImageUtils {
}
if (width == null) {
UserAgent agent = new UserAgentAdapter();
DocumentLoader loader= new DocumentLoader(agent);
DocumentLoader loader = new DocumentLoader(agent);
BridgeContext context = new BridgeContext(agent, loader);
context.setDynamic(true);
GVTBuilder builder= new GVTBuilder();
GVTBuilder builder = new GVTBuilder();
GraphicsNode root = builder.build(context, document);
var bounds = root.getPrimitiveBounds();
if (bounds != null) {

6
pom.xml
View File

@ -154,6 +154,7 @@
<oshi.version>6.4.2</oshi.version>
<google-oauth-client.version>1.34.1</google-oauth-client.version>
<apache-xmlgraphics.version>1.17</apache-xmlgraphics.version>
<drewnoakes-metadata-extractor.version>2.19.0</drewnoakes-metadata-extractor.version>
</properties>
<modules>
@ -2040,6 +2041,11 @@
<artifactId>batik-codec</artifactId>
<version>${apache-xmlgraphics.version}</version>
</dependency>
<dependency>
<groupId>com.drewnoakes</groupId>
<artifactId>metadata-extractor</artifactId>
<version>${drewnoakes-metadata-extractor.version}</version>
</dependency>
</dependencies>
</dependencyManagement>

Write Preview
Loading…
Cancel
Save