From 36a89bea918bc894a7254714efcb7a103bd8ae32 Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Thu, 14 Dec 2023 16:24:36 +0200 Subject: [PATCH] Public images --- .../main/data/upgrade/3.6.1/schema_update.sql | 30 ++++++++-- .../ThingsboardSecurityConfiguration.java | 2 +- .../server/controller/ImageController.java | 49 +++++++++++---- .../update/DefaultCacheCleanupService.java | 2 + .../queue/DefaultTbCoreConsumerService.java | 15 +++-- .../resource/DefaultTbImageService.java | 60 ++++++++++++------- .../service/resource/TbImageService.java | 2 +- .../server/dao/resource/ImageService.java | 2 + .../server/common/data/ImageExportData.java | 4 ++ .../server/common/data/TbResourceInfo.java | 11 +++- .../common/data/util/ThrowingSupplier.java | 23 +++++++ common/proto/src/main/proto/queue.proto | 7 ++- .../server/dao/model/ModelConstants.java | 2 + .../dao/model/sql/TbResourceEntity.java | 12 ++++ .../dao/model/sql/TbResourceInfoEntity.java | 12 ++++ .../server/dao/resource/BaseImageService.java | 25 +++++++- .../server/dao/resource/ImageCacheKey.java | 32 +++++++++- .../dao/resource/TbResourceInfoDao.java | 5 ++ .../sql/resource/JpaTbResourceInfoDao.java | 10 ++++ .../resource/TbResourceInfoRepository.java | 5 ++ .../resources/sql/schema-entities-idx.sql | 4 ++ .../main/resources/sql/schema-entities.sql | 2 + 22 files changed, 266 insertions(+), 50 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java diff --git a/application/src/main/data/upgrade/3.6.1/schema_update.sql b/application/src/main/data/upgrade/3.6.1/schema_update.sql index cfe16f7ff8..976c0df296 100644 --- a/application/src/main/data/upgrade/3.6.1/schema_update.sql +++ b/application/src/main/data/upgrade/3.6.1/schema_update.sql @@ -16,6 +16,28 @@ -- RESOURCES UPDATE START +ALTER TABLE resource ADD COLUMN IF NOT EXISTS descriptor varchar; +ALTER TABLE resource ADD COLUMN IF NOT EXISTS preview bytea; +ALTER TABLE resource ADD COLUMN IF NOT EXISTS external_id uuid; +ALTER TABLE resource ADD COLUMN IF NOT EXISTS is_public boolean default true; +ALTER TABLE resource ADD COLUMN IF NOT EXISTS public_key varchar(35) unique; + +CREATE INDEX IF NOT EXISTS idx_resource_etag ON resource(tenant_id, etag); +CREATE INDEX IF NOT EXISTS idx_resource_image_public_key ON resource(public_key) WHERE resource_type = 'IMAGE'; + +CREATE OR REPLACE FUNCTION generate_resource_public_key() +RETURNS text AS $$ +DECLARE + chars text := 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + result text := ''; +BEGIN + FOR i IN 1..35 LOOP + result := result || substr(chars, floor(random()*62)::int + 1, 1); + END LOOP; + RETURN result; +END; +$$ LANGUAGE plpgsql; + DO $$ BEGIN @@ -24,15 +46,13 @@ $$ ALTER TABLE resource ADD COLUMN data bytea; UPDATE resource SET data = decode(base64_data, 'base64') WHERE base64_data IS NOT NULL; ALTER TABLE resource DROP COLUMN base64_data; + ELSE + UPDATE resource SET public_key = generate_resource_public_key() WHERE resource_type = 'IMAGE' AND public_key IS NULL; END IF; END; $$; -ALTER TABLE resource ADD COLUMN IF NOT EXISTS descriptor varchar; -ALTER TABLE resource ADD COLUMN IF NOT EXISTS preview bytea; -ALTER TABLE resource ADD COLUMN IF NOT EXISTS external_id uuid; - -CREATE INDEX IF NOT EXISTS idx_resource_etag ON resource(tenant_id, etag); +DROP FUNCTION generate_resource_public_key; -- RESOURCES UPDATE END diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 6b29ba0b28..61b99e78e3 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -75,7 +75,7 @@ public class ThingsboardSecurityConfiguration { public static final String FORM_BASED_LOGIN_ENTRY_POINT = "/api/auth/login"; public static final String PUBLIC_LOGIN_ENTRY_POINT = "/api/auth/login/public"; public static final String TOKEN_REFRESH_ENTRY_POINT = "/api/auth/token"; - protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[] {"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**"}; + protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[] {"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**", "/api/images/public/**"}; public static final String TOKEN_BASED_AUTH_ENTRY_POINT = "/api/**"; public static final String WS_ENTRY_POINT = "/api/ws/**"; public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code"; diff --git a/application/src/main/java/org/thingsboard/server/controller/ImageController.java b/application/src/main/java/org/thingsboard/server/controller/ImageController.java index 815adaca5f..7dc8501188 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ImageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/ImageController.java @@ -15,8 +15,9 @@ */ package org.thingsboard.server.controller; -import com.fasterxml.jackson.core.JsonProcessingException; import io.swagger.annotations.ApiParam; +import lombok.Builder; +import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.common.data.util.ThrowingSupplier; import org.thingsboard.server.dao.resource.ImageCacheKey; import org.thingsboard.server.dao.resource.ImageService; import org.thingsboard.server.dao.service.validator.ResourceDataValidator; @@ -92,7 +94,8 @@ public class ImageController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @PostMapping("/api/image") public TbResourceInfo uploadImage(@RequestPart MultipartFile file, - @RequestPart(required = false) String title) throws Exception { + @RequestPart(required = false) String title, + @RequestPart(required = false) Boolean isPublic) throws Exception { SecurityUser user = getCurrentUser(); TbResource image = new TbResource(); image.setTenantId(user.getTenantId()); @@ -105,6 +108,7 @@ public class ImageController extends BaseController { } else { image.setTitle(file.getOriginalFilename()); } + image.setPublic(isPublic != null ? isPublic : true); image.setResourceType(ResourceType.IMAGE); ImageDescriptor descriptor = new ImageDescriptor(); descriptor.setMediaType(file.getContentType()); @@ -138,6 +142,7 @@ public class ImageController extends BaseController { @RequestBody TbResourceInfo newImageInfo) throws ThingsboardException { TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.WRITE); imageInfo.setTitle(newImageInfo.getTitle()); + imageInfo.setPublic(newImageInfo.isPublic()); return tbImageService.save(imageInfo, getCurrentUser()); } @@ -149,13 +154,28 @@ public class ImageController extends BaseController { return downloadIfChanged(type, key, etag, false); } + @GetMapping(value = "/api/images/public/{publicKey}", produces = "image/*") + public ResponseEntity downloadPublicImage(@PathVariable String publicKey, + @RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws Exception { + ImageCacheKey cacheKey = ImageCacheKey.forPublicImage(publicKey); + return downloadIfChanged(cacheKey, etag, () -> imageService.getPublicImageInfoByPublicKey(publicKey)); + } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @GetMapping(value = IMAGE_URL + "/export") public ImageExportData exportImage(@PathVariable String type, @PathVariable String key) throws Exception { TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.READ); ImageDescriptor descriptor = imageInfo.getDescriptor(ImageDescriptor.class); byte[] data = imageService.getImageData(imageInfo.getTenantId(), imageInfo.getId()); - return new ImageExportData(descriptor.getMediaType(), imageInfo.getFileName(), imageInfo.getTitle(), imageInfo.getResourceKey(), Base64Utils.encodeToString(data)); + return ImageExportData.builder() + .mediaType(descriptor.getMediaType()) + .fileName(imageInfo.getFileName()) + .title(imageInfo.getTitle()) + .resourceKey(imageInfo.getResourceKey()) + .isPublic(imageInfo.isPublic()) + .publicKey(imageInfo.getPublicKey()) + .data(Base64Utils.encodeToString(data)) + .build(); } @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @@ -172,14 +192,15 @@ public class ImageController extends BaseController { } else { image.setTitle(imageData.getFileName()); } - image.setResourceKey(imageData.getResourceKey()); image.setResourceType(ResourceType.IMAGE); + image.setResourceKey(imageData.getResourceKey()); + image.setPublic(imageData.isPublic()); + image.setPublicKey(imageData.getPublicKey()); ImageDescriptor descriptor = new ImageDescriptor(); descriptor.setMediaType(imageData.getMediaType()); image.setDescriptorValue(descriptor); image.setData(Base64Utils.decodeFromString(imageData.getData())); return tbImageService.save(image, user); - } @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") @@ -231,24 +252,28 @@ public class ImageController extends BaseController { return (result.isSuccess() ? ResponseEntity.ok() : ResponseEntity.badRequest()).body(result); } - private ResponseEntity downloadIfChanged(String type, String key, String etag, boolean preview) throws ThingsboardException, JsonProcessingException { - ImageCacheKey cacheKey = new ImageCacheKey(getTenantId(type), key, preview); + private ResponseEntity downloadIfChanged(String type, String key, String etag, boolean preview) throws Exception { + ImageCacheKey cacheKey = ImageCacheKey.forImage(getTenantId(type), key, preview); + return downloadIfChanged(cacheKey, etag, () -> checkImageInfo(type, key, Operation.READ)); + } + + private ResponseEntity downloadIfChanged(ImageCacheKey cacheKey, String etag, ThrowingSupplier imageInfoSupplier) throws Exception { if (StringUtils.isNotEmpty(etag)) { etag = StringUtils.remove(etag, '\"'); // etag is wrapped in double quotes due to HTTP specification if (etag.equals(tbImageService.getETag(cacheKey))) { return ResponseEntity.status(HttpStatus.NOT_MODIFIED).build(); } } - TenantId tenantId = getTenantId(); - TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.READ); + + TbResourceInfo imageInfo = imageInfoSupplier.get(); String fileName = imageInfo.getFileName(); ImageDescriptor descriptor = imageInfo.getDescriptor(ImageDescriptor.class); byte[] data; - if (preview) { + if (cacheKey.isPreview()) { descriptor = descriptor.getPreviewDescriptor(); - data = imageService.getImagePreview(tenantId, imageInfo.getId()); + data = imageService.getImagePreview(imageInfo.getTenantId(), imageInfo.getId()); } else { - data = imageService.getImageData(tenantId, imageInfo.getId()); + data = imageService.getImageData(imageInfo.getTenantId(), imageInfo.getId()); } tbImageService.putETag(cacheKey, descriptor.getEtag()); var result = ResponseEntity.ok() diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java index e4e1ddf427..350005536a 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java @@ -27,6 +27,7 @@ import org.springframework.stereotype.Service; import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.common.data.CacheConstants.RESOURCE_INFO_CACHE; import static org.thingsboard.server.common.data.CacheConstants.SECURITY_SETTINGS_CACHE; @RequiredArgsConstructor @@ -91,6 +92,7 @@ public class DefaultCacheCleanupService implements CacheCleanupService { case "3.6.1": log.info("Clearing cache to upgrade from version 3.6.1 to 3.6.2"); clearCacheByName(SECURITY_SETTINGS_CACHE); + clearCacheByName(RESOURCE_INFO_CACHE); break; default: //Do nothing, since cache cleanup is optional. diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index 5283713126..08d3b2325c 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -44,8 +44,10 @@ import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; +import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.common.util.ProtoUtils; +import org.thingsboard.server.dao.resource.ImageCacheKey; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto; @@ -59,6 +61,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto; import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto; import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto; +import org.thingsboard.server.gen.transport.TransportProtos.TbEntitySubEventProto; import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesDeleteProto; import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto; import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg; @@ -66,7 +69,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg; -import org.thingsboard.server.gen.transport.TransportProtos.TbEntitySubEventProto; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; import org.thingsboard.server.queue.discovery.PartitionService; @@ -83,10 +85,8 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; -import org.thingsboard.server.dao.resource.ImageCacheKey; import org.thingsboard.server.service.resource.TbImageService; import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService; -import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg; import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.state.DeviceStateService; import org.thingsboard.server.service.subscription.SubscriptionManagerService; @@ -560,8 +560,13 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService { + if (cacheKeyProto.hasResourceKey()) { + return ImageCacheKey.forImage(tenantId, cacheKeyProto.getResourceKey()); + } else { + return ImageCacheKey.forPublicImage(cacheKeyProto.getPublicKey()); + } + }).forEach(imageService::evictETags); callback.onSuccess(); } diff --git a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java index 1fbd62f88f..151ef6ce85 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java @@ -38,8 +38,11 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; @Service @Slf4j @@ -72,8 +75,11 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb } @Override - public void evictETag(ImageCacheKey imageCacheKey) { + public void evictETags(ImageCacheKey imageCacheKey) { etagCache.invalidate(imageCacheKey); + if (imageCacheKey.getPublicKey() == null) { + etagCache.invalidate(imageCacheKey.withPreview(true)); + } } @Override @@ -82,28 +88,32 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb TenantId tenantId = image.getTenantId(); try { var oldEtag = getEtag(image); + TbResourceInfo existingImage = null; if (image.getId() == null && StringUtils.isNotEmpty(image.getResourceKey())) { - var existingImage = imageService.getImageInfoByTenantIdAndKey(tenantId, image.getResourceKey()); + existingImage = imageService.getImageInfoByTenantIdAndKey(tenantId, image.getResourceKey()); if (existingImage != null) { image.setId(existingImage.getId()); } } TbResourceInfo savedImage = imageService.saveImage(image); notificationEntityService.logEntityAction(tenantId, savedImage.getId(), savedImage, actionType, user); + + List toEvict = new ArrayList<>(); if (oldEtag.isPresent()) { var newEtag = getEtag(savedImage); if (newEtag.isPresent() && !oldEtag.get().equals(newEtag.get())) { - evictETag(new ImageCacheKey(image.getTenantId(), image.getResourceKey(), false)); - evictETag(new ImageCacheKey(image.getTenantId(), image.getResourceKey(), true)); - clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder() - .setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setResourceKey(image.getResourceKey()) - .build()) - .build()); + toEvict.add(ImageCacheKey.forImage(tenantId, image.getResourceKey())); + if (image.isPublic()) { + toEvict.add(ImageCacheKey.forPublicImage(savedImage.getPublicKey())); + } } } + if (existingImage != null && image.isPublic() != existingImage.isPublic()) { + toEvict.add(ImageCacheKey.forPublicImage(image.getPublicKey())); + } + if (!toEvict.isEmpty()) { + evictFromCache(tenantId, toEvict); + } return savedImage; } catch (Exception e) { image.setData(null); @@ -145,15 +155,13 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb TbImageDeleteResult result = imageService.deleteImage(imageInfo, force); if (result.isSuccess()) { notificationEntityService.logEntityAction(tenantId, imageId, imageInfo, ActionType.DELETED, user, imageId.toString()); - evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), false)); - evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), true)); - clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder() - .setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder() - .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) - .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) - .setResourceKey(imageInfo.getResourceKey()) - .build()) - .build()); + + List toEvict = new ArrayList<>(); + toEvict.add(ImageCacheKey.forImage(tenantId, imageInfo.getResourceKey())); + if (imageInfo.isPublic()) { + toEvict.add(ImageCacheKey.forPublicImage(imageInfo.getPublicKey())); + } + evictFromCache(tenantId, toEvict); } return result; } catch (Exception e) { @@ -161,4 +169,16 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb throw e; } } + + private void evictFromCache(TenantId tenantId, List toEvict) { + toEvict.forEach(this::evictETags); + clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder() + .setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder() + .setTenantIdMSB(tenantId.getId().getMostSignificantBits()) + .setTenantIdLSB(tenantId.getId().getLeastSignificantBits()) + .addAllKeys(toEvict.stream().map(ImageCacheKey::toProto).collect(Collectors.toList())) + .build()) + .build()); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/resource/TbImageService.java b/application/src/main/java/org/thingsboard/server/service/resource/TbImageService.java index c29cc6ad3b..15f41d691d 100644 --- a/application/src/main/java/org/thingsboard/server/service/resource/TbImageService.java +++ b/application/src/main/java/org/thingsboard/server/service/resource/TbImageService.java @@ -33,6 +33,6 @@ public interface TbImageService { void putETag(ImageCacheKey imageCacheKey, String etag); - void evictETag(ImageCacheKey imageCacheKey); + void evictETags(ImageCacheKey imageCacheKey); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java index 7e5542919b..e689a48c0b 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java @@ -34,6 +34,8 @@ public interface ImageService { TbResourceInfo getImageInfoByTenantIdAndKey(TenantId tenantId, String key); + TbResourceInfo getPublicImageInfoByPublicKey(String publicKey); + PageData getImagesByTenantId(TenantId tenantId, PageLink pageLink); PageData getAllImagesByTenantId(TenantId tenantId, PageLink pageLink); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ImageExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/ImageExportData.java index d1b33f202a..d73881b70d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ImageExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ImageExportData.java @@ -16,18 +16,22 @@ package org.thingsboard.server.common.data; import io.swagger.annotations.ApiModel; +import lombok.Builder; import lombok.Data; import lombok.extern.slf4j.Slf4j; @ApiModel @Slf4j @Data +@Builder public class ImageExportData { private final String mediaType; private final String fileName; private final String title; private final String resourceKey; + private final boolean isPublic; + private final String publicKey; private final String data; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java index b8d2267663..559ac43385 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java @@ -51,6 +51,8 @@ public class TbResourceInfo extends BaseData implements HasName, H @Length(fieldName = "resourceKey") @ApiModelProperty(position = 6, value = "Resource key.", example = "19_1.0", accessMode = ApiModelProperty.AccessMode.READ_ONLY) private String resourceKey; + private boolean isPublic; + private String publicKey; @ApiModelProperty(position = 7, value = "Resource search text.", example = "19_1.0:binaryappdatacontainer", accessMode = ApiModelProperty.AccessMode.READ_ONLY) private String searchText; @@ -79,6 +81,8 @@ public class TbResourceInfo extends BaseData implements HasName, H this.resourceType = resourceInfo.resourceType; this.resourceKey = resourceInfo.resourceKey; this.searchText = resourceInfo.searchText; + this.isPublic = resourceInfo.isPublic; + this.publicKey = resourceInfo.publicKey; this.etag = resourceInfo.etag; this.fileName = resourceInfo.fileName; this.descriptor = resourceInfo.descriptor != null ? resourceInfo.descriptor.deepCopy() : null; @@ -109,7 +113,12 @@ public class TbResourceInfo extends BaseData implements HasName, H @JsonProperty(access = JsonProperty.Access.READ_ONLY) public String getLink() { if (resourceType == ResourceType.IMAGE) { - return "/api/images/" + ((tenantId == null || !tenantId.isSysTenantId()) ? "tenant" : "system") + "/" + resourceKey; // tenantId is null in case of export to git + if (isPublic) { + return "/api/images/public/" + getPublicKey(); + } else { + String type = (tenantId != null && tenantId.isSysTenantId()) ? "system" : "tenant"; // tenantId is null in case of export to git + return "/api/images/" + type + "/" + resourceKey; + } } return null; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java b/common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java new file mode 100644 index 0000000000..0e9a74dd15 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.util; + +@FunctionalInterface +public interface ThrowingSupplier { + + T get() throws Exception; + +} diff --git a/common/proto/src/main/proto/queue.proto b/common/proto/src/main/proto/queue.proto index 15e08ee905..0c7dc82ee7 100644 --- a/common/proto/src/main/proto/queue.proto +++ b/common/proto/src/main/proto/queue.proto @@ -309,7 +309,12 @@ message CoreStartupMsg { message ResourceCacheInvalidateMsg { int64 tenantIdMSB = 1; int64 tenantIdLSB = 2; - string resourceKey = 3; + repeated ImageCacheKeyProto keys = 3; +} + +message ImageCacheKeyProto { + optional string resourceKey = 1; + optional string publicKey = 2; } message LwM2MRegistrationRequestMsg { diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 21cea59ee3..c4f5047354 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -501,6 +501,8 @@ public class ModelConstants { public static final String RESOURCE_ETAG_COLUMN = "etag"; public static final String RESOURCE_DESCRIPTOR_COLUMN = "descriptor"; public static final String RESOURCE_PREVIEW_COLUMN = "preview"; + public static final String RESOURCE_IS_PUBLIC_COLUMN = "is_public"; + public static final String RESOURCE_PUBLIC_KEY_COLUMN = "public_key"; /** * Ota Package constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java index ff63994259..459e137239 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java @@ -37,8 +37,10 @@ import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_DATA_COLU import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_DESCRIPTOR_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_ETAG_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_FILE_NAME_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_IS_PUBLIC_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_KEY_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_PREVIEW_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_PUBLIC_KEY_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TENANT_ID_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TITLE_COLUMN; @@ -83,6 +85,12 @@ public class TbResourceEntity extends BaseSqlEntity { @Column(name = RESOURCE_PREVIEW_COLUMN) private byte[] preview; + @Column(name = RESOURCE_IS_PUBLIC_COLUMN) + private Boolean isPublic; + + @Column(name = RESOURCE_PUBLIC_KEY_COLUMN, unique = true, updatable = false) + private String publicKey; + @Column(name = EXTERNAL_ID_PROPERTY) private UUID externalId; @@ -106,6 +114,8 @@ public class TbResourceEntity extends BaseSqlEntity { this.etag = resource.getEtag(); this.descriptor = resource.getDescriptor(); this.preview = resource.getPreview(); + this.isPublic = resource.isPublic(); + this.publicKey = resource.getPublicKey(); this.externalId = getUuid(resource.getExternalId()); } @@ -123,6 +133,8 @@ public class TbResourceEntity extends BaseSqlEntity { resource.setEtag(etag); resource.setDescriptor(descriptor); resource.setPreview(preview); + resource.setPublic(isPublic == null || isPublic); + resource.setPublicKey(publicKey); resource.setExternalId(getEntityId(externalId, TbResourceId::new)); return resource; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java index 3a693a6301..5e488b6901 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java @@ -37,7 +37,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.EXTERNAL_ID_PROPER import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_DESCRIPTOR_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_ETAG_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_FILE_NAME_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_IS_PUBLIC_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_KEY_COLUMN; +import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_PUBLIC_KEY_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TABLE_NAME; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TENANT_ID_COLUMN; import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TITLE_COLUMN; @@ -76,6 +78,12 @@ public class TbResourceInfoEntity extends BaseSqlEntity implemen @Column(name = RESOURCE_DESCRIPTOR_COLUMN) private JsonNode descriptor; + @Column(name = RESOURCE_IS_PUBLIC_COLUMN) + private Boolean isPublic; + + @Column(name = RESOURCE_PUBLIC_KEY_COLUMN, unique = true, updatable = false) + private String publicKey; + @Column(name = EXTERNAL_ID_PROPERTY) private UUID externalId; @@ -95,6 +103,8 @@ public class TbResourceInfoEntity extends BaseSqlEntity implemen this.etag = resource.getEtag(); this.fileName = resource.getFileName(); this.descriptor = resource.getDescriptor(); + this.isPublic = resource.isPublic(); + this.publicKey = resource.getPublicKey(); this.externalId = getUuid(resource.getExternalId()); } @@ -110,6 +120,8 @@ public class TbResourceInfoEntity extends BaseSqlEntity implemen resource.setEtag(etag); resource.setFileName(fileName); resource.setDescriptor(descriptor); + resource.setPublic(isPublic == null || isPublic); + resource.setPublicKey(publicKey); resource.setExternalId(getEntityId(externalId, TbResourceId::new)); return resource; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java index f7b39166f0..e3b68b82c6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java @@ -23,6 +23,7 @@ import com.google.common.base.Strings; import lombok.Data; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.commons.lang3.tuple.Pair; @@ -149,6 +150,15 @@ public class BaseImageService extends BaseResourceService implements ImageServic image.setDescriptorValue(descriptor); image.setPreview(result.getRight()); + if (image.getId() == null) { + if (StringUtils.isEmpty(image.getPublicKey())) { + image.setPublicKey(generatePublicKey()); + } else { + if (resourceInfoDao.existsByPublicKey(ResourceType.IMAGE, image.getPublicKey())) { + image.setPublicKey(generatePublicKey()); + } + } + } log.debug("[{}] Creating image {} ('{}')", image.getTenantId(), image.getResourceKey(), image.getName()); return new TbResourceInfo(doSaveResource(image)); } @@ -194,6 +204,10 @@ public class BaseImageService extends BaseResourceService implements ImageServic return resourceKey; } + private String generatePublicKey() { + return RandomStringUtils.randomAlphanumeric(35); + } + @Override public TbResourceInfo saveImageInfo(TbResourceInfo imageInfo) { log.trace("Executing saveImageInfo [{}] [{}]", imageInfo.getTenantId(), imageInfo.getId()); @@ -206,6 +220,11 @@ public class BaseImageService extends BaseResourceService implements ImageServic return findResourceInfoByTenantIdAndKey(tenantId, ResourceType.IMAGE, key); } + @Override + public TbResourceInfo getPublicImageInfoByPublicKey(String publicKey) { + return resourceInfoDao.findPublicResourceByPublicKey(ResourceType.IMAGE, publicKey); + } + @Override public PageData getImagesByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing getImagesByTenantId [{}]", tenantId); @@ -602,7 +621,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic try { ImageCacheKey key = getKeyFromUrl(tenantId, url); if (key != null) { - var imageInfo = getImageInfoByTenantIdAndKey(key.getTenantId(), key.getKey()); + var imageInfo = getImageInfoByTenantIdAndKey(key.getTenantId(), key.getResourceKey()); if (imageInfo != null) { byte[] data = key.isPreview() ? getImagePreview(tenantId, imageInfo.getId()) : getImageData(tenantId, imageInfo.getId()); ImageDescriptor descriptor = getImageDescriptor(imageInfo, key.isPreview()); @@ -638,9 +657,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic if (imageTenantId != null) { var parts = url.split("/"); if (parts.length == 5) { - return new ImageCacheKey(imageTenantId, parts[4], false); + return ImageCacheKey.forImage(imageTenantId, parts[4], false); } else if (parts.length == 6 && "preview".equals(parts[5])) { - return new ImageCacheKey(imageTenantId, parts[4], true); + return ImageCacheKey.forImage(imageTenantId, parts[4], true); } } return null; diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/ImageCacheKey.java b/dao/src/main/java/org/thingsboard/server/dao/resource/ImageCacheKey.java index 7ee2f4a6b4..992f630978 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/ImageCacheKey.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/ImageCacheKey.java @@ -15,14 +15,44 @@ */ package org.thingsboard.server.dao.resource; +import lombok.AccessLevel; import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.With; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.transport.TransportProtos.ImageCacheKeyProto; @Data +@RequiredArgsConstructor(access = AccessLevel.PRIVATE) public class ImageCacheKey { private final TenantId tenantId; - private final String key; + private final String resourceKey; + @With private final boolean preview; + private final String publicKey; + + public static ImageCacheKey forImage(TenantId tenantId, String key, boolean preview) { + return new ImageCacheKey(tenantId, key, preview, null); + } + + public static ImageCacheKey forImage(TenantId tenantId, String key) { + return forImage(tenantId, key, false); + } + + public static ImageCacheKey forPublicImage(String publicKey) { + return new ImageCacheKey(null, null, false, publicKey); + } + + public ImageCacheKeyProto toProto() { + var msg = ImageCacheKeyProto.newBuilder(); + if (resourceKey != null) { + msg.setResourceKey(resourceKey); + } else { + msg.setPublicKey(publicKey); + } + return msg.build(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceInfoDao.java index d18ced319e..44f42391c5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/resource/TbResourceInfoDao.java @@ -41,4 +41,9 @@ public interface TbResourceInfoDao extends Dao { List findByTenantIdAndEtagAndKeyStartingWith(TenantId tenantId, String etag, String query); TbResourceInfo findSystemOrTenantImageByEtag(TenantId tenantId, ResourceType resourceType, String etag); + + boolean existsByPublicKey(ResourceType resourceType, String publicKey); + + TbResourceInfo findPublicResourceByPublicKey(ResourceType resourceType, String publicKey); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceInfoDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceInfoDao.java index d3123fe38c..501bb6b81c 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceInfoDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/resource/JpaTbResourceInfoDao.java @@ -109,4 +109,14 @@ public class JpaTbResourceInfoDao extends JpaAbstractDao 'READ'; CREATE INDEX IF NOT EXISTS idx_resource_etag ON resource(tenant_id, etag); + +CREATE INDEX IF NOT EXISTS idx_resource_etag ON resource(tenant_id, etag); + +CREATE INDEX IF NOT EXISTS idx_resource_image_public_key ON resource(public_key) WHERE resource_type = 'IMAGE'; diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 35d1d49cda..fdd6eabc39 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -718,6 +718,8 @@ CREATE TABLE IF NOT EXISTS resource ( etag varchar, descriptor varchar, preview bytea, + is_public boolean default true, + public_key varchar(35) unique, external_id uuid, CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key) );