From b047ccfd7ee257ecf52c709e4ae8c3fb8905622a Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Wed, 20 Jul 2022 17:00:21 +0300 Subject: [PATCH 01/11] firmware --- .../controller/OtaPackageController.java | 76 ++++---- .../service/action/EntityActionService.java | 18 +- .../transport/DefaultTransportApiService.java | 7 +- common/cache/pom.xml | 4 + .../cache/ota/service/FileCacheService.java | 163 ++++++++++++++++++ .../server/dao/ota/OtaPackageService.java | 2 + .../server/common/data/OtaPackage.java | 3 +- .../dao/model/sql/OtaPackageEntity.java | 7 +- .../server/dao/ota/BaseOtaPackageService.java | 79 ++++++++- .../validator/OtaPackageDataValidator.java | 21 ++- 10 files changed, 328 insertions(+), 52 deletions(-) create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index 002956924d..fa60ea3033 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -17,9 +17,11 @@ package org.thingsboard.server.controller; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.core.io.ByteArrayResource; +import org.hibernate.engine.jdbc.BlobProxy; +import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -39,15 +41,20 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.cache.ota.service.FileCacheService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; -import java.nio.ByteBuffer; +import javax.transaction.Transactional; +import java.io.InputStream; +import java.sql.Blob; +import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES; @@ -69,32 +76,35 @@ import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LI @Slf4j @RestController @TbCoreComponent +@RequiredArgsConstructor @RequestMapping("/api") public class OtaPackageController extends BaseController { public static final String OTA_PACKAGE_ID = "otaPackageId"; public static final String CHECKSUM_ALGORITHM = "checksumAlgorithm"; + private final FileCacheService fileCacheService; + @ApiOperation(value = "Download OTA Package (downloadOtaPackage)", notes = "Download OTA Package based on the provided OTA Package Id." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')") @RequestMapping(value = "/otaPackage/{otaPackageId}/download", method = RequestMethod.GET) @ResponseBody - public ResponseEntity downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) + @Transactional + public ResponseEntity downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) @PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException { checkParameter(OTA_PACKAGE_ID, strOtaPackageId); try { OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId)); - OtaPackage otaPackage = checkOtaPackageId(otaPackageId, Operation.READ); - +// OtaPackage otaPackage = checkOtaPackageId(otaPackageId, Operation.READ); + OtaPackage otaPackage = otaPackageService.findOtaPackageById(new TenantId(UUID.fromString("b04858a0-b646-11ec-8d4d-f5c10326c05e")), otaPackageId); if (otaPackage.hasUrl()) { return ResponseEntity.badRequest().build(); } - - ByteArrayResource resource = new ByteArrayResource(otaPackage.getData().array()); + InputStreamResource resource = fileCacheService.getOtaResourceById(new TenantId(UUID.fromString("b04858a0-b646-11ec-8d4d-f5c10326c05e")), otaPackageId); return ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName()) .header("x-filename", otaPackage.getFileName()) - .contentLength(resource.contentLength()) + .contentLength(otaPackage.getDataSize()) .contentType(parseMediaType(otaPackage.getContentType())) .body(resource); } catch (Exception e) { @@ -184,32 +194,12 @@ public class OtaPackageController extends BaseController { checkParameter(CHECKSUM_ALGORITHM, checksumAlgorithmStr); try { OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId)); - OtaPackageInfo info = checkOtaPackageInfoId(otaPackageId, Operation.READ); - - OtaPackage otaPackage = new OtaPackage(otaPackageId); - otaPackage.setCreatedTime(info.getCreatedTime()); - otaPackage.setTenantId(getTenantId()); - otaPackage.setDeviceProfileId(info.getDeviceProfileId()); - otaPackage.setType(info.getType()); - otaPackage.setTitle(info.getTitle()); - otaPackage.setVersion(info.getVersion()); - otaPackage.setTag(info.getTag()); - otaPackage.setAdditionalInfo(info.getAdditionalInfo()); - ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.valueOf(checksumAlgorithmStr.toUpperCase()); - byte[] bytes = file.getBytes(); if (StringUtils.isEmpty(checksum)) { - checksum = otaPackageService.generateChecksum(checksumAlgorithm, ByteBuffer.wrap(bytes)); + checksum = otaPackageService.generateChecksum(checksumAlgorithm, file.getInputStream()); } - - otaPackage.setChecksumAlgorithm(checksumAlgorithm); - otaPackage.setChecksum(checksum); - otaPackage.setFileName(file.getOriginalFilename()); - otaPackage.setContentType(file.getContentType()); - otaPackage.setData(ByteBuffer.wrap(bytes)); - otaPackage.setDataSize((long) bytes.length); - OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); + OtaPackageInfo savedOtaPackage = saveOtaPackageWithData(otaPackageId, file, checksum, checksumAlgorithm); logEntityAction(savedOtaPackage.getId(), savedOtaPackage, null, ActionType.UPDATED, null); return savedOtaPackage; } catch (Exception e) { @@ -218,6 +208,32 @@ public class OtaPackageController extends BaseController { } } + private OtaPackage saveOtaPackageWithData(OtaPackageId otaPackageId, MultipartFile file, String checksum, ChecksumAlgorithm checksumAlgorithm) throws ThingsboardException { + OtaPackageInfo info = checkOtaPackageInfoId(otaPackageId, Operation.READ); + OtaPackage otaPackage = new OtaPackage(otaPackageId); + otaPackage.setCreatedTime(info.getCreatedTime()); + otaPackage.setTenantId(info.getTenantId()); + otaPackage.setDeviceProfileId(info.getDeviceProfileId()); + otaPackage.setType(info.getType()); + otaPackage.setTitle(info.getTitle()); + otaPackage.setVersion(info.getVersion()); + otaPackage.setTag(info.getTag()); + otaPackage.setAdditionalInfo(info.getAdditionalInfo()); + otaPackage.setChecksumAlgorithm(checksumAlgorithm); + otaPackage.setChecksum(checksum); + try(InputStream inputStream = file.getInputStream()) { + otaPackage.setFileName(file.getOriginalFilename()); + otaPackage.setContentType(file.getContentType()); + otaPackage.setDataSize(file.getSize()); + Blob blob = BlobProxy.generateProxy(inputStream, file.getSize()); + otaPackage.setData(blob); + return otaPackageService.saveOtaPackage(otaPackage); + } catch (Exception e){ + log.error("Failed to save ota package {} with data {}",otaPackageId, file.getName(), e); + return null; + } + } + @ApiOperation(value = "Get OTA Package Infos (getOtaPackages)", notes = "Returns a page of OTA Package Info objects owned by tenant. " + PAGE_DATA_PARAMETERS + OTA_PACKAGE_INFO_DESCRIPTION + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH, diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index b8c0656bd2..821ecd2b35 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -22,11 +22,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.HasName; -import org.thingsboard.server.common.data.HasTenantId; -import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; @@ -159,9 +155,15 @@ public class EntityActionService { } ObjectNode entityNode; if (entity != null) { - entityNode = json.valueToTree(entity); - if (entityId.getEntityType() == EntityType.DASHBOARD) { - entityNode.put("configuration", ""); + if (entity instanceof OtaPackage) { + OtaPackage otaPackage = (OtaPackage) entity; + otaPackage.setData(null); + entityNode = json.valueToTree(otaPackage); //todo + } else { + entityNode = json.valueToTree(entity); + if (entityId.getEntityType() == EntityType.DASHBOARD) { + entityNode.put("configuration", ""); + } } } else { entityNode = json.createObjectNode(); diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 5fda933234..ae6e0a6462 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -99,6 +99,7 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.resource.TbResourceService; +import java.sql.SQLException; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -596,7 +597,11 @@ public class DefaultTransportApiService implements TransportApiService { builder.setContentType(otaPackageInfo.getContentType()); if (!otaPackageDataCache.has(otaPackageId.toString())) { OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); - otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().array()); + try { + otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().getBytes(100L, 0));//todo + } catch (SQLException e) { + log.error("ERROR", e); // todo + } } } } diff --git a/common/cache/pom.xml b/common/cache/pom.xml index 351a23ea4c..4b87f6a78e 100644 --- a/common/cache/pom.xml +++ b/common/cache/pom.xml @@ -40,6 +40,10 @@ org.thingsboard.common data + + org.thingsboard.common + dao-api + org.springframework.boot spring-boot-autoconfigure diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java new file mode 100644 index 0000000000..dd9f1eb049 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java @@ -0,0 +1,163 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.ota.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.core.io.InputStreamResource; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.id.OtaPackageId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.ota.OtaPackageService; + +import javax.annotation.PostConstruct; +import java.io.*; +import java.sql.Blob; +import java.sql.SQLException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +@Slf4j +@RequiredArgsConstructor +@Component +public class FileCacheService { + private final static String PATH = "files/%s"; + private final static String FILE_NAME_TEMPLATE = "%s.tmp"; + private final static long TEMPORARY_FILE_INACTIVITY_TIME = 300; + private final ConcurrentMap files = new ConcurrentHashMap<>(); + private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); + private final OtaPackageService otaPackageService; + + private void deleteUnusedTemporaryFiles() { + long currentTime = System.currentTimeMillis(); + List toBeDeleted = lastActivityTimes.entrySet() + .stream() + .filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + try { + toBeDeleted.forEach(otaId -> { + deleteFile(otaId.getId().toString()); + lastActivityTimes.remove(otaId); + }); + }catch (Exception e){ + log.error(",,", e); + } + log.info("Deleted {} unused temporary files", toBeDeleted.size()); + } + + private void deleteFile(String otaId) { + if (exist(otaId)) { + String fileName = String.format(FILE_NAME_TEMPLATE, otaId); + File file = new File(String.format(PATH, fileName)); + file.delete(); + } + } + + private void cleanDirectoryWithTemporaryFiles() { + File directory = new File(String.format(PATH, "")); + if (directory.isDirectory()) { + try { + FileUtils.cleanDirectory(directory); + log.info("Directory with temporary files cleaned"); + } catch (IOException e) { + log.error("Failed to clean directory with temporary files", e); + throw new RuntimeException(e); + } + } + } + + + @PostConstruct + private void createScheduledTaskToClear() { + cleanDirectoryWithTemporaryFiles(); + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 3, 3, TimeUnit.MINUTES); + } + + public InputStreamResource getOtaResourceById(TenantId tenantId, OtaPackageId otaPackageId) { + try { + String fileName = String.format(FILE_NAME_TEMPLATE, otaPackageId.getId().toString()); + if (exist(fileName)) { + lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); + return new InputStreamResource(new FileInputStream(String.format(PATH, fileName))); + } + InputStreamResource resource = load(tenantId, otaPackageId); + lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); + files.remove(otaPackageId); + return resource; + } catch (FileNotFoundException e) { + log.error("Failed to find resource for otaPackage {}", otaPackageId, e); + throw new RuntimeException(e); + } + } + + private InputStreamResource load(TenantId tenantId, OtaPackageId otaPackageId) throws FileNotFoundException { + CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(tenantId, ota)); + String fileName = String.format(FILE_NAME_TEMPLATE, otaPackageId.getId().toString()); + if (latch.getCount() == 0) { + return new InputStreamResource(new FileInputStream(String.format(PATH, fileName))); + } else { + try { + latch.await(); + return new InputStreamResource(new FileInputStream(String.format(PATH, fileName))); + } catch (InterruptedException e) { + log.error("Failed to save file {} to file system", fileName, e); + throw new RuntimeException(e); + } + } + } + + private CountDownLatch processFileSaving(TenantId tenantId, OtaPackageId otaPackageId) { + CountDownLatch latch = new CountDownLatch(1); + OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); + if (otaPackage == null) { + latch.countDown(); + log.error("Can't find otaPackage to download file {}", otaPackage); + throw new RuntimeException("No such OtaPackageId"); + } + try { + Blob data = otaPackage.getData(); + InputStream binaryStream = data.getBinaryStream(); + String fileName = String.format(FILE_NAME_TEMPLATE, otaPackageId.getId().toString()); + saveAsSystemFile(fileName, binaryStream); + latch.countDown(); + return latch; + } catch (SQLException e) { + log.error("Failed to get binary data from otaPackage {}", otaPackageId, e); + throw new RuntimeException("Failed to get binary data from database"); + } + } + + private void saveAsSystemFile(String fileName, InputStream inputStream) { + try { + File file = new File(String.format(PATH, fileName)); + FileUtils.copyInputStreamToFile(inputStream, file); + } catch (IOException e) { + log.error("Failed to copy stream to system file {}", fileName, e); + throw new RuntimeException("Failed to save file"); + } + } + + private boolean exist(String name) { + File file = new File(String.format(PATH, name)); + return file.exists(); + } +} \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index c643ae4797..8e8038cf32 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import java.io.InputStream; import java.nio.ByteBuffer; public interface OtaPackageService { @@ -51,4 +52,5 @@ public interface OtaPackageService { void deleteOtaPackagesByTenantId(TenantId tenantId); long sumDataSizeByTenantId(TenantId tenantId); + String generateChecksum(ChecksumAlgorithm checksumAlgorithm, InputStream fileData); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java index 3a17aefab8..b4957443e9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java @@ -22,6 +22,7 @@ import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.OtaPackageId; import java.nio.ByteBuffer; +import java.sql.Blob; @ApiModel @Data @@ -31,7 +32,7 @@ public class OtaPackage extends OtaPackageInfo { private static final long serialVersionUID = 3091601761339422546L; @ApiModelProperty(position = 16, value = "OTA Package data.", readOnly = true) - private transient ByteBuffer data; + private transient Blob data; public OtaPackage() { super(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java index 9d61cf52f1..f83b768e41 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java @@ -38,6 +38,7 @@ import javax.persistence.Enumerated; import javax.persistence.Lob; import javax.persistence.Table; import java.nio.ByteBuffer; +import java.sql.Blob; import java.util.UUID; import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_COLUMN; @@ -100,7 +101,7 @@ public class OtaPackageEntity extends BaseSqlEntity implements Searc @Lob @Column(name = OTA_PACKAGE_DATA_COLUMN, columnDefinition = "BINARY") - private byte[] data; + private Blob data; @Column(name = OTA_PACKAGE_DATA_SIZE_COLUMN) private Long dataSize; @@ -132,7 +133,7 @@ public class OtaPackageEntity extends BaseSqlEntity implements Searc this.contentType = otaPackage.getContentType(); this.checksumAlgorithm = otaPackage.getChecksumAlgorithm(); this.checksum = otaPackage.getChecksum(); - this.data = otaPackage.getData().array(); + this.data = otaPackage.getData(); this.dataSize = otaPackage.getDataSize(); this.additionalInfo = otaPackage.getAdditionalInfo(); } @@ -166,7 +167,7 @@ public class OtaPackageEntity extends BaseSqlEntity implements Searc otaPackage.setChecksum(checksum); otaPackage.setDataSize(dataSize); if (data != null) { - otaPackage.setData(ByteBuffer.wrap(data)); + otaPackage.setData(data); otaPackage.setHasData(true); } otaPackage.setAdditionalInfo(additionalInfo); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index ecf8d68267..6a1c14a639 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -20,6 +20,8 @@ import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.hibernate.engine.jdbc.BlobProxy; import org.hibernate.exception.ConstraintViolationException; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; @@ -40,10 +42,15 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; +import java.io.*; import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.UUID; import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -56,6 +63,8 @@ public class BaseOtaPackageService implements OtaPackageService { public static final String INCORRECT_OTA_PACKAGE_ID = "Incorrect otaPackageId "; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; + private static final int ONE_MEGA_BYTE = 1_000_000; + private final OtaPackageDao otaPackageDao; private final OtaPackageInfoDao otaPackageInfoDao; private final CacheManager cacheManager; @@ -91,15 +100,28 @@ public class BaseOtaPackageService implements OtaPackageService { @Override public OtaPackage saveOtaPackage(OtaPackage otaPackage) { log.trace("Executing saveOtaPackage [{}]", otaPackage); - otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); try { + File tempFile = saveDataToTemporaryFile(otaPackage.getData().getBinaryStream()); + otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); + try { + otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); + }catch (RuntimeException e){ + deleteTempFile(tempFile); + throw e; + } + otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); OtaPackageId otaPackageId = otaPackage.getId(); if (otaPackageId != null) { Cache cache = cacheManager.getCache(OTA_PACKAGE_CACHE); cache.evict(toOtaPackageInfoKey(otaPackageId)); otaPackageDataCache.evict(otaPackageId.toString()); } - return otaPackageDao.save(otaPackage.getTenantId(), otaPackage); + OtaPackage save = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); + deleteTempFile(tempFile); + return save; + } catch (FileNotFoundException | SQLException e){ + log.error("Failed to validate ota package {}",otaPackage.getId(), e); + throw new RuntimeException(e); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("ota_package_tenant_title_version_unq_key")) { @@ -110,6 +132,30 @@ public class BaseOtaPackageService implements OtaPackageService { } } + + public void deleteTempFile(File file) { + try { + if(file.exists()) { + FileUtils.delete(file); + log.info("System file {} was deleted", file.getName()); + } + } catch (IOException e) { + log.error("Failed to delete file {}", file.getName(), e); + } + } + + public File saveDataToTemporaryFile(InputStream inputStream){ + File path = new File("files/"); + try { + File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); + FileUtils.copyInputStreamToFile(inputStream, tempFile); + return tempFile; + }catch (IOException e){ + log.error("Failed to create temp file", e); + throw new RuntimeException("Failed to create temp file for input stream"); + } + } + @Override public String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data) { if (data == null || !data.hasArray() || data.array().length == 0) { @@ -244,4 +290,33 @@ public class BaseOtaPackageService implements OtaPackageService { return Collections.singletonList(otaPackageId); } + public String generateChecksum(ChecksumAlgorithm checksumAlgorithm, InputStream fileData) { + try { + MessageDigest md = MessageDigest.getInstance(checksumAlgorithm.name()); + return checksum(fileData, md); + } catch (NoSuchAlgorithmException e) { + log.error("No such checksum algorithm {}", checksumAlgorithm, e); + throw new RuntimeException(e); + } catch (IOException e) { + log.error("Failed to calculate checksum", e); + throw new RuntimeException(e); + } catch (Exception e) { + log.error("ff", e); + throw new RuntimeException(e); + } + } + + private String checksum(InputStream inputStream, MessageDigest md) throws IOException { + byte[] buffer = new byte[ONE_MEGA_BYTE]; + int count = 0; + while ((count = inputStream.read(buffer)) != -1) { + md.update(buffer, 0, count); + } + StringBuilder result = new StringBuilder(); + for (byte b : md.digest()) { + result.append(String.format("%02x", b)); + } + return result.toString(); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java index 8b261ad226..449d005ec1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.service.validator; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; @@ -27,9 +28,14 @@ import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; + +import java.sql.SQLException; + + import static org.thingsboard.server.common.data.EntityType.OTA_PACKAGE; @Component +@Slf4j public class OtaPackageDataValidator extends BaseOtaPackageDataValidator { @Autowired @@ -70,13 +76,14 @@ public class OtaPackageDataValidator extends BaseOtaPackageDataValidator Date: Thu, 21 Jul 2022 17:41:27 +0300 Subject: [PATCH 02/11] http sending + checksum improved --- .../sql/schema-entities-idx-psql-addon.sql | 38 + .../src/main/data/sql/schema-entities-idx.sql | 53 ++ .../src/main/data/sql/schema-entities.sql | 695 ++++++++++++++++++ .../src/main/data/sql/schema-timescale.sql | 157 ++++ .../src/main/data/sql/schema-ts-psql.sql | 339 +++++++++ .../controller/OtaPackageController.java | 9 +- .../src/main/resources/thingsboard.yml | 10 +- .../cache/ota/CaffeineOtaPackageCache.java | 8 +- .../cache/ota/service/FileCacheService.java | 148 ++-- .../server/dao/ota/OtaPackageService.java | 11 +- .../common/data/ota/ChecksumAlgorithm.java | 31 +- .../server/dao/ota/BaseOtaPackageService.java | 73 +- .../server/dao/ota/util/ChecksumUtil.java | 79 ++ .../validator/OtaPackageDataValidator.java | 7 +- .../server/dao/ota/util/ChecksumUtilTest.java | 58 ++ .../dao/service/AbstractServiceTest.java | 3 +- .../service/BaseDeviceProfileServiceTest.java | 3 +- .../dao/service/BaseDeviceServiceTest.java | 5 +- .../service/BaseOtaPackageServiceTest.java | 4 +- 19 files changed, 1571 insertions(+), 160 deletions(-) create mode 100644 application/src/main/data/sql/schema-entities-idx-psql-addon.sql create mode 100644 application/src/main/data/sql/schema-entities-idx.sql create mode 100644 application/src/main/data/sql/schema-entities.sql create mode 100644 application/src/main/data/sql/schema-timescale.sql create mode 100644 application/src/main/data/sql/schema-ts-psql.sql create mode 100644 dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java create mode 100644 dao/src/test/java/org/thingsboard/server/dao/ota/util/ChecksumUtilTest.java diff --git a/application/src/main/data/sql/schema-entities-idx-psql-addon.sql b/application/src/main/data/sql/schema-entities-idx-psql-addon.sql new file mode 100644 index 0000000000..7e15b419a8 --- /dev/null +++ b/application/src/main/data/sql/schema-entities-idx-psql-addon.sql @@ -0,0 +1,38 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- This file describes PostgreSQL-specific indexes that not supported by hsql +-- It is not a stand-alone file! Run schema-entities-idx.sql before! +-- Note: Hibernate DESC order translates to native SQL "ORDER BY .. DESC NULLS LAST" +-- While creating index PostgreSQL transforms short notation (ts DESC) to the full (DESC NULLS FIRST) +-- That difference between NULLS LAST and NULLS FIRST prevents to hit index while querying latest by ts +-- That why we need to define DESC index explicitly as (ts DESC NULLS LAST) + +CREATE INDEX IF NOT EXISTS idx_event_ts + ON public.event + (ts DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_ts + IS 'This index helps to delete events by TTL using timestamp'; + +CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des + ON public.event + (tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST) + WITH (FILLFACTOR=95); + +COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des + IS 'This index helps to open latest events on UI fast'; diff --git a/application/src/main/data/sql/schema-entities-idx.sql b/application/src/main/data/sql/schema-entities-idx.sql new file mode 100644 index 0000000000..b74d935c12 --- /dev/null +++ b/application/src/main/data/sql/schema-entities-idx.sql @@ -0,0 +1,53 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE INDEX IF NOT EXISTS idx_alarm_originator_alarm_type ON alarm(originator_id, type, start_ts DESC); + +CREATE INDEX IF NOT EXISTS idx_alarm_originator_created_time ON alarm(originator_id, created_time DESC); + +CREATE INDEX IF NOT EXISTS idx_alarm_tenant_created_time ON alarm(tenant_id, created_time DESC); + +CREATE INDEX IF NOT EXISTS idx_alarm_tenant_status_created_time ON alarm(tenant_id, status, created_time DESC); + +CREATE INDEX IF NOT EXISTS idx_alarm_tenant_alarm_type_created_time ON alarm(tenant_id, type, created_time DESC); + +CREATE INDEX IF NOT EXISTS idx_entity_alarm_created_time ON entity_alarm(tenant_id, entity_id, created_time DESC); + +CREATE INDEX IF NOT EXISTS idx_entity_alarm_alarm_id ON entity_alarm(alarm_id); + +CREATE INDEX IF NOT EXISTS idx_relation_to_id ON relation(relation_type_group, to_type, to_id); + +CREATE INDEX IF NOT EXISTS idx_relation_from_id ON relation(relation_type_group, from_type, from_id); + +CREATE INDEX IF NOT EXISTS idx_device_customer_id ON device(tenant_id, customer_id); + +CREATE INDEX IF NOT EXISTS idx_device_customer_id_and_type ON device(tenant_id, customer_id, type); + +CREATE INDEX IF NOT EXISTS idx_device_type ON device(tenant_id, type); + +CREATE INDEX IF NOT EXISTS idx_device_device_profile_id ON device(tenant_id, device_profile_id); + +CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id); + +CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); + +CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); + +CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); + +CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); + +CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); diff --git a/application/src/main/data/sql/schema-entities.sql b/application/src/main/data/sql/schema-entities.sql new file mode 100644 index 0000000000..4e2596485b --- /dev/null +++ b/application/src/main/data/sql/schema-entities.sql @@ -0,0 +1,695 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS tb_schema_settings +( + schema_version bigint NOT NULL, + CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) +); + +CREATE OR REPLACE PROCEDURE insert_tb_schema_settings() + LANGUAGE plpgsql AS +$$ +BEGIN + IF (SELECT COUNT(*) FROM tb_schema_settings) = 0 THEN + INSERT INTO tb_schema_settings (schema_version) VALUES (3003000); + END IF; +END; +$$; + +call insert_tb_schema_settings(); + +CREATE TABLE IF NOT EXISTS admin_settings ( + id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY, + created_time bigint NOT NULL, + json_value varchar, + key varchar(255) +); + +CREATE TABLE IF NOT EXISTS alarm ( + id uuid NOT NULL CONSTRAINT alarm_pkey PRIMARY KEY, + created_time bigint NOT NULL, + ack_ts bigint, + clear_ts bigint, + additional_info varchar, + end_ts bigint, + originator_id uuid, + originator_type integer, + propagate boolean, + severity varchar(255), + start_ts bigint, + status varchar(255), + tenant_id uuid, + customer_id uuid, + propagate_relation_types varchar, + type varchar(255), + propagate_to_owner boolean, + propagate_to_tenant boolean +); + +CREATE TABLE IF NOT EXISTS entity_alarm ( + tenant_id uuid NOT NULL, + entity_type varchar(32), + entity_id uuid NOT NULL, + created_time bigint NOT NULL, + alarm_type varchar(255) NOT NULL, + customer_id uuid, + alarm_id uuid, + CONSTRAINT entity_alarm_pkey PRIMARY KEY (entity_id, alarm_id), + CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS asset ( + id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + type varchar(255), + CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) +); + +CREATE TABLE IF NOT EXISTS audit_log ( + id uuid NOT NULL CONSTRAINT audit_log_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid, + customer_id uuid, + entity_id uuid, + entity_type varchar(255), + entity_name varchar(255), + user_id uuid, + user_name varchar(255), + action_type varchar(255), + action_data varchar(1000000), + action_status varchar(255), + action_failure_details varchar(1000000) +); + +CREATE TABLE IF NOT EXISTS attribute_kv ( + entity_type varchar(255), + entity_id uuid, + attribute_type varchar(255), + attribute_key varchar(255), + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json, + last_update_ts bigint, + CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_type, entity_id, attribute_type, attribute_key) +); + +CREATE TABLE IF NOT EXISTS component_descriptor ( + id uuid NOT NULL CONSTRAINT component_descriptor_pkey PRIMARY KEY, + created_time bigint NOT NULL, + actions varchar(255), + clazz varchar UNIQUE, + configuration_descriptor varchar, + name varchar(255), + scope varchar(255), + search_text varchar(255), + type varchar(255) +); + +CREATE TABLE IF NOT EXISTS customer ( + id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + address varchar, + address2 varchar, + city varchar(255), + country varchar(255), + email varchar(255), + phone varchar(255), + search_text varchar(255), + state varchar(255), + tenant_id uuid, + title varchar(255), + zip varchar(255) +); + +CREATE TABLE IF NOT EXISTS dashboard ( + id uuid NOT NULL CONSTRAINT dashboard_pkey PRIMARY KEY, + created_time bigint NOT NULL, + configuration varchar, + assigned_customers varchar(1000000), + search_text varchar(255), + tenant_id uuid, + title varchar(255), + mobile_hide boolean DEFAULT false, + mobile_order int, + image varchar(1000000) +); + +CREATE TABLE IF NOT EXISTS rule_chain ( + id uuid NOT NULL CONSTRAINT rule_chain_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + configuration varchar(10000000), + name varchar(255), + type varchar(255), + first_rule_node_id uuid, + root boolean, + debug_mode boolean, + search_text varchar(255), + tenant_id uuid +); + +CREATE TABLE IF NOT EXISTS rule_node ( + id uuid NOT NULL CONSTRAINT rule_node_pkey PRIMARY KEY, + created_time bigint NOT NULL, + rule_chain_id uuid, + additional_info varchar, + configuration varchar(10000000), + type varchar(255), + name varchar(255), + debug_mode boolean, + search_text varchar(255) +); + +CREATE TABLE IF NOT EXISTS rule_node_state ( + id uuid NOT NULL CONSTRAINT rule_node_state_pkey PRIMARY KEY, + created_time bigint NOT NULL, + rule_node_id uuid NOT NULL, + entity_type varchar(32) NOT NULL, + entity_id uuid NOT NULL, + state_data varchar(16384) NOT NULL, + CONSTRAINT rule_node_state_unq_key UNIQUE (rule_node_id, entity_id), + CONSTRAINT fk_rule_node_state_node_id FOREIGN KEY (rule_node_id) REFERENCES rule_node(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS ota_package ( + id uuid NOT NULL CONSTRAINT ota_package_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + device_profile_id uuid , + type varchar(32) NOT NULL, + title varchar(255) NOT NULL, + version varchar(255) NOT NULL, + tag varchar(255), + url varchar(255), + file_name varchar(255), + content_type varchar(255), + checksum_algorithm varchar(32), + checksum varchar(1020), + data oid, + data_size bigint, + additional_info varchar, + search_text varchar(255), + CONSTRAINT ota_package_tenant_title_version_unq_key UNIQUE (tenant_id, title, version) +); + +CREATE TABLE IF NOT EXISTS device_profile ( + id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, + created_time bigint NOT NULL, + name varchar(255), + type varchar(255), + image varchar(1000000), + transport_type varchar(255), + provision_type varchar(255), + profile_data jsonb, + description varchar, + search_text varchar(255), + is_default boolean, + tenant_id uuid, + firmware_id uuid, + software_id uuid, + default_rule_chain_id uuid, + default_dashboard_id uuid, + default_queue_name varchar(255), + provision_device_key varchar, + CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), + CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), + CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard(id), + CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package(id), + CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package(id) +); + +ALTER TABLE ota_package + ADD CONSTRAINT fk_device_profile_ota_package + FOREIGN KEY (device_profile_id) REFERENCES device_profile (id) + ON DELETE CASCADE; + +-- We will use one-to-many relation in the first release and extend this feature in case of user requests +-- CREATE TABLE IF NOT EXISTS device_profile_firmware ( +-- device_profile_id uuid NOT NULL, +-- firmware_id uuid NOT NULL, +-- CONSTRAINT device_profile_firmware_unq_key UNIQUE (device_profile_id, firmware_id), +-- CONSTRAINT fk_device_profile_firmware_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id) ON DELETE CASCADE, +-- CONSTRAINT fk_device_profile_firmware_firmware FOREIGN KEY (firmware_id) REFERENCES firmware(id) ON DELETE CASCADE, +-- ); + +CREATE TABLE IF NOT EXISTS device ( + id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + device_profile_id uuid NOT NULL, + device_data jsonb, + type varchar(255), + name varchar(255), + label varchar(255), + search_text varchar(255), + tenant_id uuid, + firmware_id uuid, + software_id uuid, + CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), + CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), + CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package(id) +); + +CREATE TABLE IF NOT EXISTS device_credentials ( + id uuid NOT NULL CONSTRAINT device_credentials_pkey PRIMARY KEY, + created_time bigint NOT NULL, + credentials_id varchar, + credentials_type varchar(255), + credentials_value varchar, + device_id uuid, + CONSTRAINT device_credentials_id_unq_key UNIQUE (credentials_id), + CONSTRAINT device_credentials_device_id_unq_key UNIQUE (device_id) +); + +CREATE TABLE IF NOT EXISTS event ( + id uuid NOT NULL CONSTRAINT event_pkey PRIMARY KEY, + created_time bigint NOT NULL, + body varchar(10000000), + entity_id uuid, + entity_type varchar(255), + event_type varchar(255), + event_uid varchar(255), + tenant_id uuid, + ts bigint NOT NULL, + CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) +); + +CREATE TABLE IF NOT EXISTS relation ( + from_id uuid, + from_type varchar(255), + to_id uuid, + to_type varchar(255), + relation_type_group varchar(255), + relation_type varchar(255), + additional_info varchar, + CONSTRAINT relation_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type) +); +-- ) PARTITION BY LIST (relation_type_group); +-- +-- CREATE TABLE other_relations PARTITION OF relation DEFAULT; +-- CREATE TABLE common_relations PARTITION OF relation FOR VALUES IN ('COMMON'); +-- CREATE TABLE alarm_relations PARTITION OF relation FOR VALUES IN ('ALARM'); +-- CREATE TABLE dashboard_relations PARTITION OF relation FOR VALUES IN ('DASHBOARD'); +-- CREATE TABLE rule_relations PARTITION OF relation FOR VALUES IN ('RULE_CHAIN', 'RULE_NODE'); + +CREATE TABLE IF NOT EXISTS tb_user ( + id uuid NOT NULL CONSTRAINT tb_user_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + authority varchar(255), + customer_id uuid, + email varchar(255) UNIQUE, + first_name varchar(255), + last_name varchar(255), + search_text varchar(255), + tenant_id uuid +); + +CREATE TABLE IF NOT EXISTS tenant_profile ( + id uuid NOT NULL CONSTRAINT tenant_profile_pkey PRIMARY KEY, + created_time bigint NOT NULL, + name varchar(255), + profile_data jsonb, + description varchar, + search_text varchar(255), + is_default boolean, + isolated_tb_core boolean, + isolated_tb_rule_engine boolean, + CONSTRAINT tenant_profile_name_unq_key UNIQUE (name) +); + +CREATE TABLE IF NOT EXISTS tenant ( + id uuid NOT NULL CONSTRAINT tenant_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + tenant_profile_id uuid NOT NULL, + address varchar, + address2 varchar, + city varchar(255), + country varchar(255), + email varchar(255), + phone varchar(255), + region varchar(255), + search_text varchar(255), + state varchar(255), + title varchar(255), + zip varchar(255), + CONSTRAINT fk_tenant_profile FOREIGN KEY (tenant_profile_id) REFERENCES tenant_profile(id) +); + +CREATE TABLE IF NOT EXISTS user_credentials ( + id uuid NOT NULL CONSTRAINT user_credentials_pkey PRIMARY KEY, + created_time bigint NOT NULL, + activate_token varchar(255) UNIQUE, + enabled boolean, + password varchar(255), + reset_token varchar(255) UNIQUE, + user_id uuid UNIQUE +); + +CREATE TABLE IF NOT EXISTS widget_type ( + id uuid NOT NULL CONSTRAINT widget_type_pkey PRIMARY KEY, + created_time bigint NOT NULL, + alias varchar(255), + bundle_alias varchar(255), + descriptor varchar(1000000), + name varchar(255), + tenant_id uuid, + image varchar(1000000), + description varchar(255) +); + +CREATE TABLE IF NOT EXISTS widgets_bundle ( + id uuid NOT NULL CONSTRAINT widgets_bundle_pkey PRIMARY KEY, + created_time bigint NOT NULL, + alias varchar(255), + search_text varchar(255), + tenant_id uuid, + title varchar(255), + image varchar(1000000), + description varchar(255) +); + +CREATE TABLE IF NOT EXISTS entity_view ( + id uuid NOT NULL CONSTRAINT entity_view_pkey PRIMARY KEY, + created_time bigint NOT NULL, + entity_id uuid, + entity_type varchar(255), + tenant_id uuid, + customer_id uuid, + type varchar(255), + name varchar(255), + keys varchar(10000000), + start_ts bigint, + end_ts bigint, + search_text varchar(255), + additional_info varchar +); + +CREATE TABLE IF NOT EXISTS ts_kv_latest +( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json, + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) +); + +CREATE TABLE IF NOT EXISTS ts_kv_dictionary +( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) +); + +CREATE TABLE IF NOT EXISTS oauth2_params ( + id uuid NOT NULL CONSTRAINT oauth2_params_pkey PRIMARY KEY, + enabled boolean, + tenant_id uuid, + created_time bigint NOT NULL +); + +CREATE TABLE IF NOT EXISTS oauth2_registration ( + id uuid NOT NULL CONSTRAINT oauth2_registration_pkey PRIMARY KEY, + oauth2_params_id uuid NOT NULL, + created_time bigint NOT NULL, + additional_info varchar, + client_id varchar(255), + client_secret varchar(2048), + authorization_uri varchar(255), + token_uri varchar(255), + scope varchar(255), + platforms varchar(255), + user_info_uri varchar(255), + user_name_attribute_name varchar(255), + jwk_set_uri varchar(255), + client_authentication_method varchar(255), + login_button_label varchar(255), + login_button_icon varchar(255), + allow_user_creation boolean, + activate_user boolean, + type varchar(31), + basic_email_attribute_key varchar(31), + basic_first_name_attribute_key varchar(31), + basic_last_name_attribute_key varchar(31), + basic_tenant_name_strategy varchar(31), + basic_tenant_name_pattern varchar(255), + basic_customer_name_pattern varchar(255), + basic_default_dashboard_name varchar(255), + basic_always_full_screen boolean, + custom_url varchar(255), + custom_username varchar(255), + custom_password varchar(255), + custom_send_token boolean, + CONSTRAINT fk_registration_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE +); + +CREATE TABLE IF NOT EXISTS oauth2_domain ( + id uuid NOT NULL CONSTRAINT oauth2_domain_pkey PRIMARY KEY, + oauth2_params_id uuid NOT NULL, + created_time bigint NOT NULL, + domain_name varchar(255), + domain_scheme varchar(31), + CONSTRAINT fk_domain_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE, + CONSTRAINT oauth2_domain_unq_key UNIQUE (oauth2_params_id, domain_name, domain_scheme) +); + +CREATE TABLE IF NOT EXISTS oauth2_mobile ( + id uuid NOT NULL CONSTRAINT oauth2_mobile_pkey PRIMARY KEY, + oauth2_params_id uuid NOT NULL, + created_time bigint NOT NULL, + pkg_name varchar(255), + app_secret varchar(2048), + CONSTRAINT fk_mobile_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE, + CONSTRAINT oauth2_mobile_unq_key UNIQUE (oauth2_params_id, pkg_name) +); + +CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( + id uuid NOT NULL CONSTRAINT oauth2_client_registration_template_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + provider_id varchar(255), + authorization_uri varchar(255), + token_uri varchar(255), + scope varchar(255), + user_info_uri varchar(255), + user_name_attribute_name varchar(255), + jwk_set_uri varchar(255), + client_authentication_method varchar(255), + type varchar(31), + basic_email_attribute_key varchar(31), + basic_first_name_attribute_key varchar(31), + basic_last_name_attribute_key varchar(31), + basic_tenant_name_strategy varchar(31), + basic_tenant_name_pattern varchar(255), + basic_customer_name_pattern varchar(255), + basic_default_dashboard_name varchar(255), + basic_always_full_screen boolean, + comment varchar, + login_button_icon varchar(255), + login_button_label varchar(255), + help_link varchar(255), + CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) +); + +-- Deprecated +CREATE TABLE IF NOT EXISTS oauth2_client_registration_info ( + id uuid NOT NULL CONSTRAINT oauth2_client_registration_info_pkey PRIMARY KEY, + enabled boolean, + created_time bigint NOT NULL, + additional_info varchar, + client_id varchar(255), + client_secret varchar(255), + authorization_uri varchar(255), + token_uri varchar(255), + scope varchar(255), + user_info_uri varchar(255), + user_name_attribute_name varchar(255), + jwk_set_uri varchar(255), + client_authentication_method varchar(255), + login_button_label varchar(255), + login_button_icon varchar(255), + allow_user_creation boolean, + activate_user boolean, + type varchar(31), + basic_email_attribute_key varchar(31), + basic_first_name_attribute_key varchar(31), + basic_last_name_attribute_key varchar(31), + basic_tenant_name_strategy varchar(31), + basic_tenant_name_pattern varchar(255), + basic_customer_name_pattern varchar(255), + basic_default_dashboard_name varchar(255), + basic_always_full_screen boolean, + custom_url varchar(255), + custom_username varchar(255), + custom_password varchar(255), + custom_send_token boolean +); + +-- Deprecated +CREATE TABLE IF NOT EXISTS oauth2_client_registration ( + id uuid NOT NULL CONSTRAINT oauth2_client_registration_pkey PRIMARY KEY, + created_time bigint NOT NULL, + domain_name varchar(255), + domain_scheme varchar(31), + client_registration_info_id uuid +); + +CREATE TABLE IF NOT EXISTS api_usage_state ( + id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid, + entity_type varchar(32), + entity_id uuid, + transport varchar(32), + db_storage varchar(32), + re_exec varchar(32), + js_exec varchar(32), + email_exec varchar(32), + sms_exec varchar(32), + alarm_exec varchar(32), + CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) +); + +CREATE TABLE IF NOT EXISTS resource ( + id uuid NOT NULL CONSTRAINT resource_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + title varchar(255) NOT NULL, + resource_type varchar(32) NOT NULL, + resource_key varchar(255) NOT NULL, + search_text varchar(255), + file_name varchar(255) NOT NULL, + data varchar, + CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key) +); + +CREATE TABLE IF NOT EXISTS edge ( + id uuid NOT NULL CONSTRAINT edge_pkey PRIMARY KEY, + created_time bigint NOT NULL, + additional_info varchar, + customer_id uuid, + root_rule_chain_id uuid, + type varchar(255), + name varchar(255), + label varchar(255), + routing_key varchar(255), + secret varchar(255), + search_text varchar(255), + tenant_id uuid, + CONSTRAINT edge_name_unq_key UNIQUE (tenant_id, name), + CONSTRAINT edge_routing_key_unq_key UNIQUE (routing_key) +); + +CREATE TABLE IF NOT EXISTS edge_event ( + id uuid NOT NULL CONSTRAINT edge_event_pkey PRIMARY KEY, + created_time bigint NOT NULL, + edge_id uuid, + edge_event_type varchar(255), + edge_event_uid varchar(255), + entity_id uuid, + edge_event_action varchar(255), + body varchar(10000000), + tenant_id uuid, + ts bigint NOT NULL +); + +CREATE TABLE IF NOT EXISTS rpc ( + id uuid NOT NULL CONSTRAINT rpc_pkey PRIMARY KEY, + created_time bigint NOT NULL, + tenant_id uuid NOT NULL, + device_id uuid NOT NULL, + expiration_time bigint NOT NULL, + request varchar(10000000) NOT NULL, + response varchar(10000000), + additional_info varchar(10000000), + status varchar(255) NOT NULL +); + +CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl( + IN regular_events_start_ts bigint, + IN regular_events_end_ts bigint, + IN debug_events_start_ts bigint, + IN debug_events_end_ts bigint, + INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_deleted_count bigint DEFAULT 0; + debug_ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF regular_events_start_ts > 0 AND regular_events_end_ts > 0 THEN + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE id in (SELECT id from event WHERE ts > %L::bigint AND ts < %L::bigint AND ' || + '(event_type != %L::varchar AND event_type != %L::varchar)) RETURNING *) ' || + 'SELECT count(*) FROM deleted', regular_events_start_ts, regular_events_end_ts, + 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; + END IF; + IF debug_events_start_ts > 0 AND debug_events_end_ts > 0 THEN + EXECUTE format( + 'WITH deleted AS (DELETE FROM event WHERE id in (SELECT id from event WHERE ts > %L::bigint AND ts < %L::bigint AND ' || + '(event_type = %L::varchar OR event_type = %L::varchar)) RETURNING *) ' || + 'SELECT count(*) FROM deleted', debug_events_start_ts, debug_events_end_ts, + 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; + END IF; + RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; + RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; + deleted := ttl_deleted_count + debug_ttl_deleted_count; +END +$$; + +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; + + +CREATE OR REPLACE PROCEDURE cleanup_edge_events_by_ttl(IN ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + ttl_ts bigint; + ttl_deleted_count bigint DEFAULT 0; +BEGIN + IF ttl > 0 THEN + ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; + EXECUTE format( + 'WITH deleted AS (DELETE FROM edge_event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into ttl_deleted_count; + END IF; + RAISE NOTICE 'Edge events removed by ttl: %', ttl_deleted_count; + deleted := ttl_deleted_count; +END +$$; diff --git a/application/src/main/data/sql/schema-timescale.sql b/application/src/main/data/sql/schema-timescale.sql new file mode 100644 index 0000000000..81b71f1a03 --- /dev/null +++ b/application/src/main/data/sql/schema-timescale.sql @@ -0,0 +1,157 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; + +CREATE TABLE IF NOT EXISTS ts_kv ( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json, + CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) +); + +CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) +); + +CREATE TABLE IF NOT EXISTS ts_kv_latest ( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json, + CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) +); + +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT device.id as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT asset.id as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT customer.id as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record uuid; + customer_id_record uuid; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN + OPEN tenant_cursor; + FETCH tenant_cursor INTO tenant_id_record; + WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; + END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; + END IF; + FOR customer_id_record IN + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + customer_id_record, 'TTL') INTO customer_ttl; + IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; + ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; + END IF; + END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + END IF; + END LOOP; + FETCH tenant_cursor INTO tenant_id_record; + END LOOP; +END +$$; diff --git a/application/src/main/data/sql/schema-ts-psql.sql b/application/src/main/data/sql/schema-ts-psql.sql new file mode 100644 index 0000000000..c95ac4cf0f --- /dev/null +++ b/application/src/main/data/sql/schema-ts-psql.sql @@ -0,0 +1,339 @@ +-- +-- Copyright © 2016-2022 The Thingsboard Authors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +CREATE TABLE IF NOT EXISTS ts_kv +( + entity_id uuid NOT NULL, + key int NOT NULL, + ts bigint NOT NULL, + bool_v boolean, + str_v varchar(10000000), + long_v bigint, + dbl_v double precision, + json_v json, + CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) +) PARTITION BY RANGE (ts); + +CREATE TABLE IF NOT EXISTS ts_kv_dictionary +( + key varchar(255) NOT NULL, + key_id serial UNIQUE, + CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) +); + +CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + max_tenant_ttl bigint; + max_customer_ttl bigint; + max_ttl bigint; + date timestamp; + partition_by_max_ttl_date varchar; + partition_by_max_ttl_month varchar; + partition_by_max_ttl_day varchar; + partition_by_max_ttl_year varchar; + partition varchar; + partition_year integer; + partition_month integer; + partition_day integer; + +BEGIN + SELECT max(attribute_kv.long_v) + FROM tenant + INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_tenant_ttl; + SELECT max(attribute_kv.long_v) + FROM customer + INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id + WHERE attribute_kv.attribute_key = 'TTL' + into max_customer_ttl; + max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl); + if max_ttl IS NOT NULL AND max_ttl > 0 THEN + date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl); + partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date); + RAISE NOTICE 'Date by max ttl: %', date; + RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date; + IF partition_by_max_ttl_date IS NOT NULL THEN + CASE + WHEN partition_type = 'DAYS' THEN + partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + partition_by_max_ttl_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5); + WHEN partition_type = 'MONTHS' THEN + partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); + ELSE + partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); + END CASE; + IF partition_by_max_ttl_year IS NULL THEN + RAISE NOTICE 'Failed to remove partitions by max ttl date due to partition_by_max_ttl_year is null!'; + ELSE + IF partition_type = 'YEARS' THEN + FOR partition IN SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename like 'ts_kv_' || '%' + AND tablename != 'ts_kv_latest' + AND tablename != 'ts_kv_dictionary' + AND tablename != 'ts_kv_indefinite' + AND tablename != partition_by_max_ttl_date + LOOP + partition_year := SPLIT_PART(partition, '_', 3)::integer; + IF partition_year < partition_by_max_ttl_year::integer THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition; + EXECUTE format('DROP TABLE IF EXISTS %I', partition); + deleted := deleted + 1; + END IF; + END LOOP; + ELSE + IF partition_type = 'MONTHS' THEN + IF partition_by_max_ttl_month IS NULL THEN + RAISE NOTICE 'Failed to remove months partitions by max ttl date due to partition_by_max_ttl_month is null!'; + ELSE + FOR partition IN SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename like 'ts_kv_' || '%' + AND tablename != 'ts_kv_latest' + AND tablename != 'ts_kv_dictionary' + AND tablename != 'ts_kv_indefinite' + AND tablename != partition_by_max_ttl_date + LOOP + partition_year := SPLIT_PART(partition, '_', 3)::integer; + IF partition_year > partition_by_max_ttl_year::integer THEN + RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; + CONTINUE; + ELSE + IF partition_year < partition_by_max_ttl_year::integer THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition; + EXECUTE format('DROP TABLE IF EXISTS %I', partition); + deleted := deleted + 1; + ELSE + partition_month := SPLIT_PART(partition, '_', 4)::integer; + IF partition_year = partition_by_max_ttl_year::integer THEN + IF partition_month >= partition_by_max_ttl_month::integer THEN + RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; + CONTINUE; + ELSE + RAISE NOTICE 'Partition to delete by max ttl: %', partition; + EXECUTE format('DROP TABLE IF EXISTS %I', partition); + deleted := deleted + 1; + END IF; + END IF; + END IF; + END IF; + END LOOP; + END IF; + ELSE + IF partition_type = 'DAYS' THEN + IF partition_by_max_ttl_month IS NULL THEN + RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_month is null!'; + ELSE + IF partition_by_max_ttl_day IS NULL THEN + RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_day is null!'; + ELSE + FOR partition IN SELECT tablename + FROM pg_tables + WHERE schemaname = 'public' + AND tablename like 'ts_kv_' || '%' + AND tablename != 'ts_kv_latest' + AND tablename != 'ts_kv_dictionary' + AND tablename != 'ts_kv_indefinite' + AND tablename != partition_by_max_ttl_date + LOOP + partition_year := SPLIT_PART(partition, '_', 3)::integer; + IF partition_year > partition_by_max_ttl_year::integer THEN + RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; + CONTINUE; + ELSE + IF partition_year < partition_by_max_ttl_year::integer THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition; + EXECUTE format('DROP TABLE IF EXISTS %I', partition); + deleted := deleted + 1; + ELSE + partition_month := SPLIT_PART(partition, '_', 4)::integer; + IF partition_month > partition_by_max_ttl_month::integer THEN + RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; + CONTINUE; + ELSE + IF partition_month < partition_by_max_ttl_month::integer THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition; + EXECUTE format('DROP TABLE IF EXISTS %I', partition); + deleted := deleted + 1; + ELSE + partition_day := SPLIT_PART(partition, '_', 5)::integer; + IF partition_day >= partition_by_max_ttl_day::integer THEN + RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; + CONTINUE; + ELSE + IF partition_day < partition_by_max_ttl_day::integer THEN + RAISE NOTICE 'Partition to delete by max ttl: %', partition; + EXECUTE format('DROP TABLE IF EXISTS %I', partition); + deleted := deleted + 1; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + END LOOP; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; + END IF; +END +$$; + +CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS +$$ +BEGIN + CASE + WHEN partition_type = 'DAYS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd'); + WHEN partition_type = 'MONTHS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM'); + WHEN partition_type = 'YEARS' THEN + partition := 'ts_kv_' || to_char(date, 'yyyy'); + ELSE + partition := NULL; + END CASE; + IF partition IS NOT NULL THEN + IF NOT EXISTS(SELECT + FROM pg_tables + WHERE schemaname = 'public' + AND tablename = partition) THEN + partition := NULL; + RAISE NOTICE 'Failed to found partition by ttl'; + END IF; + END IF; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS +$$ +BEGIN + uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || + '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT device.id as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT asset.id as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, + OUT deleted bigint) AS +$$ +BEGIN + EXECUTE format( + 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT customer.id as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', + tenant_id, customer_id, ttl) into deleted; +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, + IN system_ttl bigint, INOUT deleted bigint) + LANGUAGE plpgsql AS +$$ +DECLARE + tenant_cursor CURSOR FOR select tenant.id as tenant_id + from tenant; + tenant_id_record uuid; + customer_id_record uuid; + tenant_ttl bigint; + customer_ttl bigint; + deleted_for_entities bigint; + tenant_ttl_ts bigint; + customer_ttl_ts bigint; +BEGIN + OPEN tenant_cursor; + FETCH tenant_cursor INTO tenant_id_record; + WHILE FOUND + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + tenant_id_record, 'TTL') INTO tenant_ttl; + if tenant_ttl IS NULL THEN + tenant_ttl := system_ttl; + END IF; + IF tenant_ttl > 0 THEN + tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; + deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; + END IF; + FOR customer_id_record IN + SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record + LOOP + EXECUTE format( + 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', + customer_id_record, 'TTL') INTO customer_ttl; + IF customer_ttl IS NULL THEN + customer_ttl_ts := tenant_ttl_ts; + ELSE + IF customer_ttl > 0 THEN + customer_ttl_ts := + (EXTRACT(EPOCH FROM current_timestamp) * 1000 - + customer_ttl::bigint * 1000)::bigint; + END IF; + END IF; + IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN + deleted_for_entities := + delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; + deleted_for_entities := + delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, + customer_id_record, + customer_ttl_ts); + deleted := deleted + deleted_for_entities; + RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; + END IF; + END LOOP; + FETCH tenant_cursor INTO tenant_id_record; + END LOOP; +END +$$; diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index fa60ea3033..719d56510d 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.cache.ota.service.FileCacheService; import org.thingsboard.server.service.security.permission.Operation; @@ -95,12 +96,12 @@ public class OtaPackageController extends BaseController { checkParameter(OTA_PACKAGE_ID, strOtaPackageId); try { OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId)); -// OtaPackage otaPackage = checkOtaPackageId(otaPackageId, Operation.READ); - OtaPackage otaPackage = otaPackageService.findOtaPackageById(new TenantId(UUID.fromString("b04858a0-b646-11ec-8d4d-f5c10326c05e")), otaPackageId); + OtaPackage otaPackage = checkOtaPackageId(otaPackageId, Operation.READ); if (otaPackage.hasUrl()) { return ResponseEntity.badRequest().build(); } - InputStreamResource resource = fileCacheService.getOtaResourceById(new TenantId(UUID.fromString("b04858a0-b646-11ec-8d4d-f5c10326c05e")), otaPackageId); + InputStream dataStream = otaPackageService.getOtaDataStream(getTenantId(), otaPackageId); + InputStreamResource resource = new InputStreamResource(dataStream); return ResponseEntity.ok() .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName()) .header("x-filename", otaPackage.getFileName()) @@ -197,7 +198,7 @@ public class OtaPackageController extends BaseController { ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.valueOf(checksumAlgorithmStr.toUpperCase()); if (StringUtils.isEmpty(checksum)) { - checksum = otaPackageService.generateChecksum(checksumAlgorithm, file.getInputStream()); + checksum = ChecksumUtil.generateChecksum(checksumAlgorithm, file.getInputStream()); } OtaPackageInfo savedOtaPackage = saveOtaPackageWithData(otaPackageId, file, checksum, checksumAlgorithm); logEntityAction(savedOtaPackage.getId(), savedOtaPackage, null, ActionType.UPDATED, null); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 03c3bbd9f9..4c181c0218 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -505,8 +505,8 @@ spring.resources.chain: content: enabled: "true" -spring.servlet.multipart.max-file-size: "50MB" -spring.servlet.multipart.max-request-size: "50MB" +spring.servlet.multipart.max-file-size: "2000MB" +spring.servlet.multipart.max-request-size: "2000MB" spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true" spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" @@ -526,9 +526,9 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" - username: "${SPRING_DATASOURCE_USERNAME:postgres}" - password: "${SPRING_DATASOURCE_PASSWORD:postgres}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/board}" + username: "${SPRING_DATASOURCE_USERNAME:myuser}" + password: "${SPRING_DATASOURCE_PASSWORD:mypass}" hikari: maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:16}" diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java index 7ea7b18f43..5c4e5c2e66 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java index dd9f1eb049..485360b2d0 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,12 +18,9 @@ package org.thingsboard.server.cache.ota.service; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.springframework.core.io.InputStreamResource; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.ota.OtaPackageService; import javax.annotation.PostConstruct; import java.io.*; @@ -31,6 +28,7 @@ import java.sql.Blob; import java.sql.SQLException; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.*; import java.util.stream.Collectors; @@ -38,12 +36,40 @@ import java.util.stream.Collectors; @RequiredArgsConstructor @Component public class FileCacheService { - private final static String PATH = "files/%s"; - private final static String FILE_NAME_TEMPLATE = "%s.tmp"; - private final static long TEMPORARY_FILE_INACTIVITY_TIME = 300; + @Value("${java.io.tmpdir}/ota/") + private String PATH; + private final static String FILE_PATH_TEMPLATE = "%s"; + private final static long TEMPORARY_FILE_INACTIVITY_TIME = 300;//todo increase time private final ConcurrentMap files = new ConcurrentHashMap<>(); private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); - private final OtaPackageService otaPackageService; + + @PostConstruct + private void createScheduledTaskToClear() { + createTempDirectoryIfNotExist(); + cleanDirectoryWithTemporaryFiles(); + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 3, 3, TimeUnit.MINUTES); //todo increase time + } + + private void createTempDirectoryIfNotExist() { + File directory = new File(PATH); + if(!directory.exists()){ + directory.mkdir(); + } + } + + private void cleanDirectoryWithTemporaryFiles() { + File directory = new File(PATH); + if (directory.isDirectory()) { + try { + FileUtils.cleanDirectory(directory); + log.info("Directory with temporary files cleaned"); + } catch (IOException e) { + log.error("Failed to clean directory with temporary files", e); + throw new RuntimeException(e); + } + } + } private void deleteUnusedTemporaryFiles() { long currentTime = System.currentTimeMillis(); @@ -57,67 +83,47 @@ public class FileCacheService { deleteFile(otaId.getId().toString()); lastActivityTimes.remove(otaId); }); - }catch (Exception e){ + } catch (Exception e) { log.error(",,", e); } log.info("Deleted {} unused temporary files", toBeDeleted.size()); } private void deleteFile(String otaId) { - if (exist(otaId)) { - String fileName = String.format(FILE_NAME_TEMPLATE, otaId); - File file = new File(String.format(PATH, fileName)); - file.delete(); - } + String fileName = PATH + otaId; + File file = new File(String.format(FILE_PATH_TEMPLATE, fileName)); + deleteFile(file); } - private void cleanDirectoryWithTemporaryFiles() { - File directory = new File(String.format(PATH, "")); - if (directory.isDirectory()) { - try { - FileUtils.cleanDirectory(directory); - log.info("Directory with temporary files cleaned"); - } catch (IOException e) { - log.error("Failed to clean directory with temporary files", e); - throw new RuntimeException(e); + public void deleteFile(File file) { + try { + if (file.exists()) { + FileUtils.delete(file); + log.info("System file {} was deleted", file.getName()); } + } catch (IOException e) { + log.error("Failed to delete file {}", file.getName(), e); } } - - @PostConstruct - private void createScheduledTaskToClear() { - cleanDirectoryWithTemporaryFiles(); - ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 3, 3, TimeUnit.MINUTES); - } - - public InputStreamResource getOtaResourceById(TenantId tenantId, OtaPackageId otaPackageId) { - try { - String fileName = String.format(FILE_NAME_TEMPLATE, otaPackageId.getId().toString()); - if (exist(fileName)) { - lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); - return new InputStreamResource(new FileInputStream(String.format(PATH, fileName))); - } - InputStreamResource resource = load(tenantId, otaPackageId); + public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException { + String fileName = PATH + otaPackageId.getId(); + if (exist(fileName)) { lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); - files.remove(otaPackageId); - return resource; - } catch (FileNotFoundException e) { - log.error("Failed to find resource for otaPackage {}", otaPackageId, e); - throw new RuntimeException(e); + return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); } + throw new FileNotFoundException(); } - private InputStreamResource load(TenantId tenantId, OtaPackageId otaPackageId) throws FileNotFoundException { - CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(tenantId, ota)); - String fileName = String.format(FILE_NAME_TEMPLATE, otaPackageId.getId().toString()); + private InputStream load(OtaPackageId otaPackageId, Blob data) throws FileNotFoundException { + CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data)); + String fileName = PATH + otaPackageId.getId(); if (latch.getCount() == 0) { - return new InputStreamResource(new FileInputStream(String.format(PATH, fileName))); + return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); } else { try { latch.await(); - return new InputStreamResource(new FileInputStream(String.format(PATH, fileName))); + return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); } catch (InterruptedException e) { log.error("Failed to save file {} to file system", fileName, e); throw new RuntimeException(e); @@ -125,18 +131,11 @@ public class FileCacheService { } } - private CountDownLatch processFileSaving(TenantId tenantId, OtaPackageId otaPackageId) { + private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) { CountDownLatch latch = new CountDownLatch(1); - OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); - if (otaPackage == null) { - latch.countDown(); - log.error("Can't find otaPackage to download file {}", otaPackage); - throw new RuntimeException("No such OtaPackageId"); - } try { - Blob data = otaPackage.getData(); InputStream binaryStream = data.getBinaryStream(); - String fileName = String.format(FILE_NAME_TEMPLATE, otaPackageId.getId().toString()); + String fileName =PATH + otaPackageId.getId(); saveAsSystemFile(fileName, binaryStream); latch.countDown(); return latch; @@ -148,7 +147,7 @@ public class FileCacheService { private void saveAsSystemFile(String fileName, InputStream inputStream) { try { - File file = new File(String.format(PATH, fileName)); + File file = new File(String.format(FILE_PATH_TEMPLATE, fileName)); FileUtils.copyInputStreamToFile(inputStream, file); } catch (IOException e) { log.error("Failed to copy stream to system file {}", fileName, e); @@ -157,7 +156,28 @@ public class FileCacheService { } private boolean exist(String name) { - File file = new File(String.format(PATH, name)); + File file = new File(String.format(FILE_PATH_TEMPLATE, name)); return file.exists(); } + + public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) { + try { + return load(otaPackageId, data); + } catch (FileNotFoundException e) { + log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e); + throw new RuntimeException(e); + } + } + + public File saveDataTemporaryFile(InputStream inputStream) { + File path = new File(PATH); + try { + File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); + FileUtils.copyInputStreamToFile(inputStream, tempFile); + return tempFile; + } catch (IOException e) { + log.error("Failed to create temp file", e); + throw new RuntimeException("Failed to create temp file for input stream"); + } + } } \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index 8e8038cf32..8ce5e64066 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -52,5 +52,6 @@ public interface OtaPackageService { void deleteOtaPackagesByTenantId(TenantId tenantId); long sumDataSizeByTenantId(TenantId tenantId); - String generateChecksum(ChecksumAlgorithm checksumAlgorithm, InputStream fileData); + + InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java b/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java index 6c764ccd3f..73477b5cc6 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,12 +15,21 @@ */ package org.thingsboard.server.common.data.ota; +import lombok.Getter; + public enum ChecksumAlgorithm { - MD5, - SHA256, - SHA384, - SHA512, - CRC32, - MURMUR3_32, - MURMUR3_128 + MD5("MD5"), + SHA256("SHA-256"), + SHA384("SHA-384"), + SHA512("SHA-512"), + CRC32("CRC32"), + MURMUR3_32("MURMUR3-32"), + MURMUR3_128("MURMUR3_128"); + + @Getter + final String name; + + ChecksumAlgorithm(String name) { + this.name = name; + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 6a1c14a639..ffa6906cd0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -28,6 +28,7 @@ import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.ota.OtaPackageDataCache; +import org.thingsboard.server.cache.ota.service.FileCacheService; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; @@ -42,6 +43,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; +import javax.transaction.Transactional; import java.io.*; import java.nio.ByteBuffer; import java.security.MessageDigest; @@ -62,15 +64,13 @@ import static org.thingsboard.server.dao.service.Validator.validatePageLink; public class BaseOtaPackageService implements OtaPackageService { public static final String INCORRECT_OTA_PACKAGE_ID = "Incorrect otaPackageId "; public static final String INCORRECT_TENANT_ID = "Incorrect tenantId "; - - private static final int ONE_MEGA_BYTE = 1_000_000; - private final OtaPackageDao otaPackageDao; private final OtaPackageInfoDao otaPackageInfoDao; private final CacheManager cacheManager; private final OtaPackageDataCache otaPackageDataCache; private final DataValidator otaPackageInfoValidator; private final DataValidator otaPackageValidator; + private final FileCacheService fileCacheService; @Override public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl) { @@ -101,12 +101,12 @@ public class BaseOtaPackageService implements OtaPackageService { public OtaPackage saveOtaPackage(OtaPackage otaPackage) { log.trace("Executing saveOtaPackage [{}]", otaPackage); try { - File tempFile = saveDataToTemporaryFile(otaPackage.getData().getBinaryStream()); + File tempFile= fileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream()); otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); try { otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); }catch (RuntimeException e){ - deleteTempFile(tempFile); + fileCacheService.deleteFile(tempFile); throw e; } otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); @@ -117,7 +117,7 @@ public class BaseOtaPackageService implements OtaPackageService { otaPackageDataCache.evict(otaPackageId.toString()); } OtaPackage save = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); - deleteTempFile(tempFile); + fileCacheService.deleteFile(tempFile); return save; } catch (FileNotFoundException | SQLException e){ log.error("Failed to validate ota package {}",otaPackage.getId(), e); @@ -131,31 +131,6 @@ public class BaseOtaPackageService implements OtaPackageService { } } } - - - public void deleteTempFile(File file) { - try { - if(file.exists()) { - FileUtils.delete(file); - log.info("System file {} was deleted", file.getName()); - } - } catch (IOException e) { - log.error("Failed to delete file {}", file.getName(), e); - } - } - - public File saveDataToTemporaryFile(InputStream inputStream){ - File path = new File("files/"); - try { - File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); - FileUtils.copyInputStreamToFile(inputStream, tempFile); - return tempFile; - }catch (IOException e){ - log.error("Failed to create temp file", e); - throw new RuntimeException("Failed to create temp file for input stream"); - } - } - @Override public String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data) { if (data == null || !data.hasArray() || data.array().length == 0) { @@ -290,33 +265,17 @@ public class BaseOtaPackageService implements OtaPackageService { return Collections.singletonList(otaPackageId); } - public String generateChecksum(ChecksumAlgorithm checksumAlgorithm, InputStream fileData) { + @Transactional + public InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId){ try { - MessageDigest md = MessageDigest.getInstance(checksumAlgorithm.name()); - return checksum(fileData, md); - } catch (NoSuchAlgorithmException e) { - log.error("No such checksum algorithm {}", checksumAlgorithm, e); - throw new RuntimeException(e); - } catch (IOException e) { - log.error("Failed to calculate checksum", e); - throw new RuntimeException(e); - } catch (Exception e) { - log.error("ff", e); - throw new RuntimeException(e); - } - } - - private String checksum(InputStream inputStream, MessageDigest md) throws IOException { - byte[] buffer = new byte[ONE_MEGA_BYTE]; - int count = 0; - while ((count = inputStream.read(buffer)) != -1) { - md.update(buffer, 0, count); - } - StringBuilder result = new StringBuilder(); - for (byte b : md.digest()) { - result.append(String.format("%02x", b)); + return fileCacheService.getOtaDataStream(otaPackageId); + }catch (FileNotFoundException e){ + OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); + if (otaPackage == null) { + log.error("Can't find otaPackage to download file {}", otaPackage); + throw new RuntimeException("No such OtaPackageId"); + } + return fileCacheService.loadOtaData(otaPackageId,otaPackage.getData()); } - return result.toString(); } - } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java b/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java new file mode 100644 index 0000000000..9d3c7469e2 --- /dev/null +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java @@ -0,0 +1,79 @@ +package org.thingsboard.server.dao.ota.util; + +import com.datastax.oss.driver.shaded.guava.common.io.ByteStreams; +import com.google.common.hash.Funnels; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; + +import java.io.IOException; +import java.io.InputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Base64; +import java.util.zip.CRC32; + +@Slf4j +public class ChecksumUtil { + private static final int ONE_MEGA_BYTE = 1_000_000; + + public static String generateChecksum(ChecksumAlgorithm checksumAlgorithm, InputStream fileData) { + try { + switch (checksumAlgorithm) { + case CRC32: + return checksumCRC32(fileData); + case MURMUR3_128: + return checksumMurmur3_128(fileData); + case MURMUR3_32: + return checksumMurmur3_32(fileData); + default: + MessageDigest md = MessageDigest.getInstance(checksumAlgorithm.getName()); + return checksum(fileData, md); + } + } catch (NoSuchAlgorithmException e) { + log.error("No such checksum algorithm {}", checksumAlgorithm, e); + throw new RuntimeException(e); + } catch (Exception e) { + log.error("Failed to calculate checksum", e); + throw new RuntimeException(e); + } + } + + private static String checksum(InputStream inputStream, MessageDigest md) throws IOException { + byte[] buffer = new byte[ONE_MEGA_BYTE]; + int count = 0; + while ((count = inputStream.read(buffer)) != -1) { + md.update(buffer, 0, count); + } + StringBuilder result = new StringBuilder(); + for (byte b : md.digest()) { + result.append(String.format("%02x", b)); + } + return result.toString(); + } + + private static String checksumCRC32(InputStream inputStream) throws IOException { + CRC32 crc = new CRC32(); + byte[] buffer = new byte[ONE_MEGA_BYTE]; + int count = 0; + while ((count = inputStream.read(buffer)) != -1) { + crc.update(buffer, 0, count); + } + return Long.toHexString(crc.getValue()); + } + + private static String checksumMurmur3_32(InputStream stream) throws IOException { + Hasher hasher = Hashing.murmur3_32().newHasher(); + ByteStreams.copy(stream, Funnels.asOutputStream(hasher)); + String hash = new String(hasher.hash().asBytes()); + return new String(Base64.getEncoder().encode(hash.getBytes())); + } + + private static String checksumMurmur3_128(InputStream stream) throws IOException { + Hasher hasher = Hashing.murmur3_128().newHasher(); + ByteStreams.copy(stream, Funnels.asOutputStream(hasher)); + String hash = new String(hasher.hash().asBytes()); + return new String(Base64.getEncoder().encode(hash.getBytes())); + } +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java index 449d005ec1..e55b5e333a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; @@ -41,10 +42,6 @@ public class OtaPackageDataValidator extends BaseOtaPackageDataValidatorChecksumUtil.generateChecksum(MURMUR3_32, inputStream)); + } + + @Test + void testMurmur3_128Checksum(){ + assertDoesNotThrow(()->ChecksumUtil.generateChecksum(MURMUR3_128, inputStream)); + } + +} \ No newline at end of file diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index c1ecafda6e..03783cf833 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.lang3.RandomStringUtils; +import org.hibernate.engine.jdbc.BlobProxy; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -268,7 +269,7 @@ public abstract class AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(ByteBuffer.wrap(new byte[]{1})); + firmware.setData(BlobProxy.generateProxy(new byte[]{1})); firmware.setDataSize(1L); return firmware; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java index 4e0b0e7dfc..400286c13e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java @@ -19,6 +19,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; +import org.hibernate.engine.jdbc.BlobProxy; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -110,7 +111,7 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(ByteBuffer.wrap(new byte[]{1})); + firmware.setData(BlobProxy.generateProxy(new byte[]{1})); firmware.setDataSize(1L); OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java index b4c0ff77dd..283ac8fd0e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; import org.apache.commons.lang3.RandomStringUtils; +import org.hibernate.engine.jdbc.BlobProxy; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -200,7 +201,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(ByteBuffer.wrap(new byte[]{1})); + firmware.setData(BlobProxy.generateProxy(new byte[]{1})); firmware.setDataSize(1L); OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); @@ -235,7 +236,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(ByteBuffer.wrap(new byte[]{1})); + firmware.setData(BlobProxy.generateProxy(new byte[]{1})); firmware.setDataSize(1L); OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 7dedf14bab..006392bb45 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; import org.apache.commons.lang3.RandomStringUtils; +import org.hibernate.engine.jdbc.BlobProxy; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -40,6 +41,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import javax.validation.ValidationException; import java.nio.ByteBuffer; +import java.sql.Blob; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -56,7 +58,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { private static final ChecksumAlgorithm CHECKSUM_ALGORITHM = ChecksumAlgorithm.SHA256; private static final String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"; private static final long DATA_SIZE = 1L; - private static final ByteBuffer DATA = ByteBuffer.wrap(new byte[]{(int) DATA_SIZE}); + private static final Blob DATA = BlobProxy.generateProxy(new byte[]{(int) DATA_SIZE}); private static final String URL = "http://firmware.test.org"; private final IdComparator idComparator = new IdComparator<>(); From 67561bf256b0c99f4caa11c3e050b84bc9105330 Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Fri, 22 Jul 2022 17:34:15 +0300 Subject: [PATCH 03/11] improved fileSaving --- .../controller/OtaPackageController.java | 26 +-- common/cache/pom.xml | 6 + .../ota/service/BaseFileCacheService.java | 137 +++++++++++++ .../cache/ota/service/FileCacheService.java | 188 +----------------- .../ota/service/TemporaryFileCleaner.java | 88 ++++++++ .../ota/service/BaseFileCacheServiceTest.java | 101 ++++++++++ .../server/dao/ota/OtaPackageService.java | 2 + .../server/dao/ota/BaseOtaPackageService.java | 32 ++- .../service/BaseOtaPackageServiceTest.java | 46 +++-- 9 files changed, 409 insertions(+), 217 deletions(-) create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java create mode 100644 common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index 719d56510d..062b98028b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -21,9 +21,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hibernate.engine.jdbc.BlobProxy; -import org.springframework.core.io.InputStreamResource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.ResponseEntity; +import org.springframework.core.io.FileSystemResource; +import org.springframework.http.*; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -41,21 +40,20 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.cache.ota.service.FileCacheService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import javax.transaction.Transactional; import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.sql.Blob; -import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES; @@ -84,14 +82,13 @@ public class OtaPackageController extends BaseController { public static final String OTA_PACKAGE_ID = "otaPackageId"; public static final String CHECKSUM_ALGORITHM = "checksumAlgorithm"; - private final FileCacheService fileCacheService; @ApiOperation(value = "Download OTA Package (downloadOtaPackage)", notes = "Download OTA Package based on the provided OTA Package Id." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')") @RequestMapping(value = "/otaPackage/{otaPackageId}/download", method = RequestMethod.GET) @ResponseBody @Transactional - public ResponseEntity downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) + public ResponseEntity downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) @PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException { checkParameter(OTA_PACKAGE_ID, strOtaPackageId); try { @@ -100,18 +97,21 @@ public class OtaPackageController extends BaseController { if (otaPackage.hasUrl()) { return ResponseEntity.badRequest().build(); } - InputStream dataStream = otaPackageService.getOtaDataStream(getTenantId(), otaPackageId); - InputStreamResource resource = new InputStreamResource(dataStream); + FileSystemResource resource = new FileSystemResource(otaPackageService.getOtaDataFile(getTenantId(), otaPackageId)); + String fileName= URLEncoder.encode(otaPackage.getFileName(), StandardCharsets.UTF_8.toString()); return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName()) - .header("x-filename", otaPackage.getFileName()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename*=UTF-8''" + fileName) + .header("x-filename", fileName) .contentLength(otaPackage.getDataSize()) - .contentType(parseMediaType(otaPackage.getContentType())) + .contentType(new MediaType(parseMediaType(otaPackage.getContentType()), StandardCharsets.UTF_8)) .body(resource); } catch (Exception e) { throw handleException(e); } } + private static String encodeFileName(String fileName) { + return URLEncoder.encode(fileName, StandardCharsets.UTF_8).replace("+", "%20"); + } @ApiOperation(value = "Get OTA Package Info (getOtaPackageInfoById)", notes = "Fetch the OTA Package Info object based on the provided OTA Package Id. " + diff --git a/common/cache/pom.xml b/common/cache/pom.xml index 4b87f6a78e..c912c4eede 100644 --- a/common/cache/pom.xml +++ b/common/cache/pom.xml @@ -95,6 +95,12 @@ awaitility test + + org.hibernate + hibernate-core + 5.4.32.Final + test + diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java new file mode 100644 index 0000000000..2f2d91a950 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.ota.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.OtaPackageId; + +import java.io.*; +import java.sql.Blob; +import java.sql.SQLException; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +@RequiredArgsConstructor +@Component +public class BaseFileCacheService implements FileCacheService { + @Value("${java.io.tmpdir}/ota/") + private String PATH; + + private final TemporaryFileCleaner fileCleaner; + private final ConcurrentMap files = new ConcurrentHashMap<>(); + + + public File saveDataTemporaryFile(InputStream inputStream) { + File path = new File(PATH); + try { + File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); + FileUtils.copyInputStreamToFile(inputStream, tempFile); + return tempFile; + } catch (IOException e) { + log.error("Failed to create temp file", e); + throw new RuntimeException("Failed to create temp file for input stream"); + } + } + + public File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException { + String fileName = PATH + otaPackageId.getId(); + if (exist(fileName)) { + fileCleaner.updateFileUsageStatus(otaPackageId); + return new File(fileName); + } + throw new FileNotFoundException(); + } + + public File loadToFile(OtaPackageId otaPackageId, Blob data) { + if (otaPackageId == null || data == null) { + log.error("Received null variables: {}", otaPackageId == null ? "otaPackageId" : "data"); + throw new NullPointerException(); + } + CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data)); + String fileName = PATH + otaPackageId.getId(); + if (latch.getCount() == 0) { + return new File(fileName); + } else { + try { + latch.await(); + return new File(fileName); + } catch (InterruptedException e) { + log.error("Failed to save file {} to file system", fileName, e); + throw new RuntimeException(e); + } + } + } + + public void deleteFile(File file) { + try { + if (file.exists()) { + FileUtils.delete(file); + log.info("System file {} was deleted", file.getName()); + } + } catch (IOException e) { + log.error("Failed to delete file {}", file.getName(), e); + } + } + + public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException { + return new FileInputStream(getOtaDataFile(otaPackageId)); + } + + public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) { + try { + return new FileInputStream(loadToFile(otaPackageId, data)); + } catch (FileNotFoundException e) { + log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e); + throw new RuntimeException(e); + } + } + + private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) { + CountDownLatch latch = new CountDownLatch(1); + try { + InputStream binaryStream = data.getBinaryStream(); + String fileName = PATH + otaPackageId.getId(); + saveAsSystemFile(fileName, binaryStream); + latch.countDown(); + return latch; + } catch (SQLException e) { + log.error("Failed to get binary data from otaPackage {}", otaPackageId, e); + throw new RuntimeException("Failed to get binary data from database"); + } + } + + private void saveAsSystemFile(String fileName, InputStream inputStream) { + try { + File file = new File(fileName); + FileUtils.copyInputStreamToFile(inputStream, file); + } catch (IOException e) { + log.error("Failed to copy stream to system file {}", fileName, e); + throw new RuntimeException("Failed to save file"); + } + } + + private boolean exist(String name) { + File file = new File(name); + return file.exists(); + } +} \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java index 485360b2d0..b022932c4f 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java @@ -1,183 +1,17 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.cache.ota.service; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.OtaPackageId; -import javax.annotation.PostConstruct; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.sql.Blob; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.*; -import java.util.stream.Collectors; -@Slf4j -@RequiredArgsConstructor -@Component -public class FileCacheService { - @Value("${java.io.tmpdir}/ota/") - private String PATH; - private final static String FILE_PATH_TEMPLATE = "%s"; - private final static long TEMPORARY_FILE_INACTIVITY_TIME = 300;//todo increase time - private final ConcurrentMap files = new ConcurrentHashMap<>(); - private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); - - @PostConstruct - private void createScheduledTaskToClear() { - createTempDirectoryIfNotExist(); - cleanDirectoryWithTemporaryFiles(); - ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 3, 3, TimeUnit.MINUTES); //todo increase time - } - - private void createTempDirectoryIfNotExist() { - File directory = new File(PATH); - if(!directory.exists()){ - directory.mkdir(); - } - } - - private void cleanDirectoryWithTemporaryFiles() { - File directory = new File(PATH); - if (directory.isDirectory()) { - try { - FileUtils.cleanDirectory(directory); - log.info("Directory with temporary files cleaned"); - } catch (IOException e) { - log.error("Failed to clean directory with temporary files", e); - throw new RuntimeException(e); - } - } - } - - private void deleteUnusedTemporaryFiles() { - long currentTime = System.currentTimeMillis(); - List toBeDeleted = lastActivityTimes.entrySet() - .stream() - .filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - try { - toBeDeleted.forEach(otaId -> { - deleteFile(otaId.getId().toString()); - lastActivityTimes.remove(otaId); - }); - } catch (Exception e) { - log.error(",,", e); - } - log.info("Deleted {} unused temporary files", toBeDeleted.size()); - } - - private void deleteFile(String otaId) { - String fileName = PATH + otaId; - File file = new File(String.format(FILE_PATH_TEMPLATE, fileName)); - deleteFile(file); - } - - public void deleteFile(File file) { - try { - if (file.exists()) { - FileUtils.delete(file); - log.info("System file {} was deleted", file.getName()); - } - } catch (IOException e) { - log.error("Failed to delete file {}", file.getName(), e); - } - } - - public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException { - String fileName = PATH + otaPackageId.getId(); - if (exist(fileName)) { - lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); - return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); - } - throw new FileNotFoundException(); - } - - private InputStream load(OtaPackageId otaPackageId, Blob data) throws FileNotFoundException { - CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data)); - String fileName = PATH + otaPackageId.getId(); - if (latch.getCount() == 0) { - return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); - } else { - try { - latch.await(); - return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); - } catch (InterruptedException e) { - log.error("Failed to save file {} to file system", fileName, e); - throw new RuntimeException(e); - } - } - } - - private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) { - CountDownLatch latch = new CountDownLatch(1); - try { - InputStream binaryStream = data.getBinaryStream(); - String fileName =PATH + otaPackageId.getId(); - saveAsSystemFile(fileName, binaryStream); - latch.countDown(); - return latch; - } catch (SQLException e) { - log.error("Failed to get binary data from otaPackage {}", otaPackageId, e); - throw new RuntimeException("Failed to get binary data from database"); - } - } - - private void saveAsSystemFile(String fileName, InputStream inputStream) { - try { - File file = new File(String.format(FILE_PATH_TEMPLATE, fileName)); - FileUtils.copyInputStreamToFile(inputStream, file); - } catch (IOException e) { - log.error("Failed to copy stream to system file {}", fileName, e); - throw new RuntimeException("Failed to save file"); - } - } - - private boolean exist(String name) { - File file = new File(String.format(FILE_PATH_TEMPLATE, name)); - return file.exists(); - } - - public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) { - try { - return load(otaPackageId, data); - } catch (FileNotFoundException e) { - log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e); - throw new RuntimeException(e); - } - } - - public File saveDataTemporaryFile(InputStream inputStream) { - File path = new File(PATH); - try { - File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); - FileUtils.copyInputStreamToFile(inputStream, tempFile); - return tempFile; - } catch (IOException e) { - log.error("Failed to create temp file", e); - throw new RuntimeException("Failed to create temp file for input stream"); - } - } -} \ No newline at end of file +public interface FileCacheService { + void deleteFile(File file); + InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException ; + InputStream loadOtaData(OtaPackageId otaPackageId, Blob data); + File loadToFile(OtaPackageId otaPackageId, Blob data); + File saveDataTemporaryFile(InputStream inputStream); + File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException; +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java new file mode 100644 index 0000000000..dc943c83d9 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java @@ -0,0 +1,88 @@ +package org.thingsboard.server.cache.ota.service; + +import lombok.extern.apachecommons.CommonsLog; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.OtaPackageId; + +import javax.annotation.PostConstruct; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +@Slf4j +@Component +public class TemporaryFileCleaner { + @Value("${java.io.tmpdir}/ota/") + private String PATH; + private final static long TEMPORARY_FILE_INACTIVITY_TIME = 900_000; + private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); + + public void updateFileUsageStatus(OtaPackageId otaPackageId){ + lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); + } + + @PostConstruct + public void createScheduledTaskToClear() { + createTempDirectoryIfNotExist(); + cleanDirectoryWithTemporaryFiles(); + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 10, 10, TimeUnit.MINUTES); + } + + private void createTempDirectoryIfNotExist() { + File directory = new File(PATH); + if (!directory.exists()) { + directory.mkdir(); + } + } + + private void cleanDirectoryWithTemporaryFiles() { + File directory = new File(PATH); + if (directory.isDirectory()) { + try { + FileUtils.cleanDirectory(directory); + log.info("Directory with temporary files cleaned"); + } catch (IOException e) { + log.error("Failed to clean directory with temporary files", e); + throw new RuntimeException(e); + } + } + } + + private void deleteUnusedTemporaryFiles() { + long currentTime = System.currentTimeMillis(); + List toBeDeleted = lastActivityTimes.entrySet() + .stream() + .filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + try { + toBeDeleted.forEach(otaId -> { + deleteFile(otaId.getId().toString()); + lastActivityTimes.remove(otaId); + }); + } catch (Exception e) { + log.error("Failed to delete unused files", e); + } + log.info("Deleted {} unused temporary files", toBeDeleted.size()); + } + + private void deleteFile(String otaId) { + String fileName = PATH + otaId; + File file = new File(fileName); + try { + if (file.exists()) { + FileUtils.delete(file); + log.info("System file {} was deleted", file.getName()); + } + } catch (IOException e) { + log.error("Failed to delete file {}", file.getName(), e); + } + } +} diff --git a/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java new file mode 100644 index 0000000000..20cd1dc47b --- /dev/null +++ b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java @@ -0,0 +1,101 @@ +package org.thingsboard.server.cache.ota.service; + +import lombok.SneakyThrows; +import org.hibernate.engine.jdbc.BlobProxy; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.thingsboard.server.common.data.id.OtaPackageId; + +import java.io.*; +import java.security.MessageDigest; +import java.sql.Blob; +import java.util.Objects; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + + +class BaseFileCacheServiceTest { + private final static String PATH = "/home/anastasiia/IdeaProjects/thingsboard/common/cache"; + private final static String FILE_FILLING = "Hello, testing environment"; + private static final int ONE_MEGA_BYTE = 1_000_000; + private final static OtaPackageId OTA_PACKAGE_ID = new OtaPackageId(UUID.randomUUID()); + private final static Blob DATA = BlobProxy.generateProxy(FILE_FILLING.getBytes()); + File file; + BaseFileCacheService baseFileCacheService = new BaseFileCacheService(new TemporaryFileCleaner()); + + @BeforeEach + void setUp() { + file = Mockito.mock(File.class); +// fileCacheService = Mockito.mock(FileCacheService.class); + } + + @Test + void testNotThrowExceptionsWhenDeleteFile() { + Mockito.when(file.exists()).thenReturn(true); + assertDoesNotThrow(() -> baseFileCacheService.deleteFile(new File("/download/alarm_rule.js"))); + } + + @Test + void testDataSavingWithNullBlob() { + assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, null)); + } + + @Test + void testDataSavingWithNullOtaPackageId() { + assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(null, DATA)); + } + + @Test + @SneakyThrows + void testMultiSavingDataToFile() { + File directory = new File(PATH); + int beginning = Objects.requireNonNull(directory.list()).length; + Thread thread1 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA)); + Thread thread2 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA)); + thread1.start(); + thread2.start(); + thread1.join(); + thread2.join(); + File directory1 = new File(PATH); + int ending = Objects.requireNonNull(directory1.list()).length; + assertEquals(1, ending - beginning); + } + + @Test + @SneakyThrows + void testCorrectDataSavingToFile() { + String sha256 = calculateChecksumSHA256(new ByteArrayInputStream(FILE_FILLING.getBytes())); + File file = baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA); + assertEquals(sha256, calculateChecksumSHA256(new FileInputStream(file))); + } + + @Test + @SneakyThrows + void testNotThrowExceptionWhenCreatingTemporaryFiles() { + BaseFileCacheService mock = Mockito.mock(BaseFileCacheService.class); + assertDoesNotThrow(() -> mock.saveDataTemporaryFile(DATA.getBinaryStream())); + } + + @Test + @SneakyThrows + void testGettingStreamWhenFileNotExist(){ + assertThrows(FileNotFoundException.class, ()->baseFileCacheService.getOtaDataStream(OTA_PACKAGE_ID)); + } + + @SneakyThrows + String calculateChecksumSHA256(InputStream stream) { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] buffer = new byte[ONE_MEGA_BYTE]; + int count = 0; + while ((count = stream.read(buffer)) != -1) { + md.update(buffer, 0, count); + } + StringBuilder result = new StringBuilder(); + for (byte b : md.digest()) { + result.append(String.format("%02x", b)); + } + return result.toString(); + } +} \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index 8ce5e64066..c0c5b64a1f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import java.io.File; import java.io.InputStream; import java.nio.ByteBuffer; @@ -54,4 +55,5 @@ public interface OtaPackageService { long sumDataSizeByTenantId(TenantId tenantId); InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId); + File getOtaDataFile (TenantId tenantId, OtaPackageId otaPackageId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index ffa6906cd0..435479932a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -20,7 +20,6 @@ import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.hibernate.engine.jdbc.BlobProxy; import org.hibernate.exception.ConstraintViolationException; import org.springframework.cache.Cache; @@ -28,7 +27,7 @@ import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.ota.OtaPackageDataCache; -import org.thingsboard.server.cache.ota.service.FileCacheService; +import org.thingsboard.server.cache.ota.service.BaseFileCacheService; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; @@ -46,13 +45,10 @@ import org.thingsboard.server.dao.service.PaginatedRemover; import javax.transaction.Transactional; import java.io.*; import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.UUID; import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -70,7 +66,7 @@ public class BaseOtaPackageService implements OtaPackageService { private final OtaPackageDataCache otaPackageDataCache; private final DataValidator otaPackageInfoValidator; private final DataValidator otaPackageValidator; - private final FileCacheService fileCacheService; + private final BaseFileCacheService baseFileCacheService; @Override public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl) { @@ -101,12 +97,12 @@ public class BaseOtaPackageService implements OtaPackageService { public OtaPackage saveOtaPackage(OtaPackage otaPackage) { log.trace("Executing saveOtaPackage [{}]", otaPackage); try { - File tempFile= fileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream()); + File tempFile= baseFileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream()); otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); try { otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); }catch (RuntimeException e){ - fileCacheService.deleteFile(tempFile); + baseFileCacheService.deleteFile(tempFile); throw e; } otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); @@ -117,7 +113,7 @@ public class BaseOtaPackageService implements OtaPackageService { otaPackageDataCache.evict(otaPackageId.toString()); } OtaPackage save = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); - fileCacheService.deleteFile(tempFile); + baseFileCacheService.deleteFile(tempFile); return save; } catch (FileNotFoundException | SQLException e){ log.error("Failed to validate ota package {}",otaPackage.getId(), e); @@ -268,14 +264,28 @@ public class BaseOtaPackageService implements OtaPackageService { @Transactional public InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId){ try { - return fileCacheService.getOtaDataStream(otaPackageId); + return baseFileCacheService.getOtaDataStream(otaPackageId); }catch (FileNotFoundException e){ OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); if (otaPackage == null) { log.error("Can't find otaPackage to download file {}", otaPackage); throw new RuntimeException("No such OtaPackageId"); } - return fileCacheService.loadOtaData(otaPackageId,otaPackage.getData()); + return baseFileCacheService.loadOtaData(otaPackageId,otaPackage.getData()); + } + } + + @Transactional + public File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId){ + try { + return baseFileCacheService.getOtaDataFile(otaPackageId); + }catch (FileNotFoundException e){ + OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); + if (otaPackage == null) { + log.error("Can't find otaPackage to download file {}", otaPackage); + throw new RuntimeException("No such OtaPackageId"); + } + return baseFileCacheService.loadToFile(otaPackageId,otaPackage.getData()); } } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 006392bb45..1e0b0340c0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,19 +18,10 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; import org.apache.commons.lang3.RandomStringUtils; import org.hibernate.engine.jdbc.BlobProxy; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.rules.ExpectedException; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.OtaPackageInfo; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; @@ -38,15 +29,20 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.util.ChecksumUtil; import javax.validation.ValidationException; -import java.nio.ByteBuffer; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.sql.Blob; import java.util.ArrayList; import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @@ -697,6 +693,24 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { otaPackageService.saveOtaPackageInfo(firmwareInfo, true); } + @Test + public void testGettingCorrectStreamWithOtaData() { + OtaPackage firmware = createFirmware(tenantId, "24687846"); + InputStream stream = otaPackageService.getOtaDataStream(tenantId, firmware.getId()); + assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, stream)); + } + + @Test + public void testGettingCorrectFileWithOtaData() { + OtaPackage firmware = createFirmware(tenantId, "24687846"); + File file = otaPackageService.getOtaDataFile(tenantId, firmware.getId()); + try { + assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, new FileInputStream(file))); + } catch (FileNotFoundException e){ + throw new RuntimeException(e); + } + } + private OtaPackage createFirmware(TenantId tenantId, String version) { OtaPackage firmware = new OtaPackage(); firmware.setTenantId(tenantId); From 49f700eba173221ccdb998b2176ffd8352580420 Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Mon, 25 Jul 2022 15:18:35 +0300 Subject: [PATCH 04/11] cleaned file download controller --- .../server/controller/OtaPackageController.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index 062b98028b..c1aadc238b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -98,20 +98,16 @@ public class OtaPackageController extends BaseController { return ResponseEntity.badRequest().build(); } FileSystemResource resource = new FileSystemResource(otaPackageService.getOtaDataFile(getTenantId(), otaPackageId)); - String fileName= URLEncoder.encode(otaPackage.getFileName(), StandardCharsets.UTF_8.toString()); return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename*=UTF-8''" + fileName) - .header("x-filename", fileName) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName()) + .header("x-filename", otaPackage.getFileName()) .contentLength(otaPackage.getDataSize()) - .contentType(new MediaType(parseMediaType(otaPackage.getContentType()), StandardCharsets.UTF_8)) + .contentType(parseMediaType(otaPackage.getContentType())) .body(resource); } catch (Exception e) { throw handleException(e); } } - private static String encodeFileName(String fileName) { - return URLEncoder.encode(fileName, StandardCharsets.UTF_8).replace("+", "%20"); - } @ApiOperation(value = "Get OTA Package Info (getOtaPackageInfoById)", notes = "Fetch the OTA Package Info object based on the provided OTA Package Id. " + From 2e2174248e05dafd5e57c26392c79f193fcaa93a Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Mon, 8 Aug 2022 14:14:01 +0300 Subject: [PATCH 05/11] First CR --- .../sql/schema-entities-idx-psql-addon.sql | 38 - .../src/main/data/sql/schema-entities-idx.sql | 53 -- .../src/main/data/sql/schema-entities.sql | 695 ------------------ .../src/main/data/sql/schema-timescale.sql | 157 ---- .../src/main/data/sql/schema-ts-psql.sql | 339 --------- .../controller/OtaPackageController.java | 62 +- .../service/action/EntityActionService.java | 27 +- .../transport/DefaultTransportApiService.java | 8 +- .../src/main/resources/thingsboard.yml | 13 +- common/cache/pom.xml | 15 +- .../cache/ota/files/BaseFileCacheService.java | 100 +++ .../cache/ota/files/FileCacheService.java | 14 + .../TemporaryFileCleaner.java | 49 +- .../ota/service/BaseFileCacheService.java | 137 ---- .../cache/ota/service/FileCacheService.java | 17 - .../ota/service/BaseFileCacheServiceTest.java | 47 +- .../server/dao/ota/OtaPackageService.java | 7 +- .../server/common/data/OtaPackage.java | 18 +- .../thingsboard/common/util/JacksonUtil.java | 1 + .../dao/model/sql/OtaPackageEntity.java | 49 +- .../server/dao/ota/BaseOtaPackageService.java | 112 +-- .../server/dao/ota/util/ChecksumUtil.java | 8 +- .../validator/OtaPackageDataValidator.java | 24 +- .../server/dao/ota/util/ChecksumUtilTest.java | 36 +- .../dao/service/AbstractServiceTest.java | 3 +- .../service/BaseDeviceProfileServiceTest.java | 3 +- .../dao/service/BaseDeviceServiceTest.java | 5 +- .../service/BaseOtaPackageServiceTest.java | 14 +- 28 files changed, 306 insertions(+), 1745 deletions(-) delete mode 100644 application/src/main/data/sql/schema-entities-idx-psql-addon.sql delete mode 100644 application/src/main/data/sql/schema-entities-idx.sql delete mode 100644 application/src/main/data/sql/schema-entities.sql delete mode 100644 application/src/main/data/sql/schema-timescale.sql delete mode 100644 application/src/main/data/sql/schema-ts-psql.sql create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java rename common/cache/src/main/java/org/thingsboard/server/cache/ota/{service => files}/TemporaryFileCleaner.java (63%) delete mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java delete mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java diff --git a/application/src/main/data/sql/schema-entities-idx-psql-addon.sql b/application/src/main/data/sql/schema-entities-idx-psql-addon.sql deleted file mode 100644 index 7e15b419a8..0000000000 --- a/application/src/main/data/sql/schema-entities-idx-psql-addon.sql +++ /dev/null @@ -1,38 +0,0 @@ --- --- Copyright © 2016-2022 The Thingsboard Authors --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - --- This file describes PostgreSQL-specific indexes that not supported by hsql --- It is not a stand-alone file! Run schema-entities-idx.sql before! --- Note: Hibernate DESC order translates to native SQL "ORDER BY .. DESC NULLS LAST" --- While creating index PostgreSQL transforms short notation (ts DESC) to the full (DESC NULLS FIRST) --- That difference between NULLS LAST and NULLS FIRST prevents to hit index while querying latest by ts --- That why we need to define DESC index explicitly as (ts DESC NULLS LAST) - -CREATE INDEX IF NOT EXISTS idx_event_ts - ON public.event - (ts DESC NULLS LAST) - WITH (FILLFACTOR=95); - -COMMENT ON INDEX public.idx_event_ts - IS 'This index helps to delete events by TTL using timestamp'; - -CREATE INDEX IF NOT EXISTS idx_event_tenant_entity_type_entity_event_type_created_time_des - ON public.event - (tenant_id ASC, entity_type ASC, entity_id ASC, event_type ASC, created_time DESC NULLS LAST) - WITH (FILLFACTOR=95); - -COMMENT ON INDEX public.idx_event_tenant_entity_type_entity_event_type_created_time_des - IS 'This index helps to open latest events on UI fast'; diff --git a/application/src/main/data/sql/schema-entities-idx.sql b/application/src/main/data/sql/schema-entities-idx.sql deleted file mode 100644 index b74d935c12..0000000000 --- a/application/src/main/data/sql/schema-entities-idx.sql +++ /dev/null @@ -1,53 +0,0 @@ --- --- Copyright © 2016-2022 The Thingsboard Authors --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -CREATE INDEX IF NOT EXISTS idx_alarm_originator_alarm_type ON alarm(originator_id, type, start_ts DESC); - -CREATE INDEX IF NOT EXISTS idx_alarm_originator_created_time ON alarm(originator_id, created_time DESC); - -CREATE INDEX IF NOT EXISTS idx_alarm_tenant_created_time ON alarm(tenant_id, created_time DESC); - -CREATE INDEX IF NOT EXISTS idx_alarm_tenant_status_created_time ON alarm(tenant_id, status, created_time DESC); - -CREATE INDEX IF NOT EXISTS idx_alarm_tenant_alarm_type_created_time ON alarm(tenant_id, type, created_time DESC); - -CREATE INDEX IF NOT EXISTS idx_entity_alarm_created_time ON entity_alarm(tenant_id, entity_id, created_time DESC); - -CREATE INDEX IF NOT EXISTS idx_entity_alarm_alarm_id ON entity_alarm(alarm_id); - -CREATE INDEX IF NOT EXISTS idx_relation_to_id ON relation(relation_type_group, to_type, to_id); - -CREATE INDEX IF NOT EXISTS idx_relation_from_id ON relation(relation_type_group, from_type, from_id); - -CREATE INDEX IF NOT EXISTS idx_device_customer_id ON device(tenant_id, customer_id); - -CREATE INDEX IF NOT EXISTS idx_device_customer_id_and_type ON device(tenant_id, customer_id, type); - -CREATE INDEX IF NOT EXISTS idx_device_type ON device(tenant_id, type); - -CREATE INDEX IF NOT EXISTS idx_device_device_profile_id ON device(tenant_id, device_profile_id); - -CREATE INDEX IF NOT EXISTS idx_asset_customer_id ON asset(tenant_id, customer_id); - -CREATE INDEX IF NOT EXISTS idx_asset_customer_id_and_type ON asset(tenant_id, customer_id, type); - -CREATE INDEX IF NOT EXISTS idx_asset_type ON asset(tenant_id, type); - -CREATE INDEX IF NOT EXISTS idx_attribute_kv_by_key_and_last_update_ts ON attribute_kv(entity_id, attribute_key, last_update_ts desc); - -CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_id_and_created_time ON audit_log(tenant_id, created_time); - -CREATE INDEX IF NOT EXISTS idx_rpc_tenant_id_device_id ON rpc(tenant_id, device_id); diff --git a/application/src/main/data/sql/schema-entities.sql b/application/src/main/data/sql/schema-entities.sql deleted file mode 100644 index 4e2596485b..0000000000 --- a/application/src/main/data/sql/schema-entities.sql +++ /dev/null @@ -1,695 +0,0 @@ --- --- Copyright © 2016-2022 The Thingsboard Authors --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -CREATE TABLE IF NOT EXISTS tb_schema_settings -( - schema_version bigint NOT NULL, - CONSTRAINT tb_schema_settings_pkey PRIMARY KEY (schema_version) -); - -CREATE OR REPLACE PROCEDURE insert_tb_schema_settings() - LANGUAGE plpgsql AS -$$ -BEGIN - IF (SELECT COUNT(*) FROM tb_schema_settings) = 0 THEN - INSERT INTO tb_schema_settings (schema_version) VALUES (3003000); - END IF; -END; -$$; - -call insert_tb_schema_settings(); - -CREATE TABLE IF NOT EXISTS admin_settings ( - id uuid NOT NULL CONSTRAINT admin_settings_pkey PRIMARY KEY, - created_time bigint NOT NULL, - json_value varchar, - key varchar(255) -); - -CREATE TABLE IF NOT EXISTS alarm ( - id uuid NOT NULL CONSTRAINT alarm_pkey PRIMARY KEY, - created_time bigint NOT NULL, - ack_ts bigint, - clear_ts bigint, - additional_info varchar, - end_ts bigint, - originator_id uuid, - originator_type integer, - propagate boolean, - severity varchar(255), - start_ts bigint, - status varchar(255), - tenant_id uuid, - customer_id uuid, - propagate_relation_types varchar, - type varchar(255), - propagate_to_owner boolean, - propagate_to_tenant boolean -); - -CREATE TABLE IF NOT EXISTS entity_alarm ( - tenant_id uuid NOT NULL, - entity_type varchar(32), - entity_id uuid NOT NULL, - created_time bigint NOT NULL, - alarm_type varchar(255) NOT NULL, - customer_id uuid, - alarm_id uuid, - CONSTRAINT entity_alarm_pkey PRIMARY KEY (entity_id, alarm_id), - CONSTRAINT fk_entity_alarm_id FOREIGN KEY (alarm_id) REFERENCES alarm(id) ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS asset ( - id uuid NOT NULL CONSTRAINT asset_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - type varchar(255), - CONSTRAINT asset_name_unq_key UNIQUE (tenant_id, name) -); - -CREATE TABLE IF NOT EXISTS audit_log ( - id uuid NOT NULL CONSTRAINT audit_log_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid, - customer_id uuid, - entity_id uuid, - entity_type varchar(255), - entity_name varchar(255), - user_id uuid, - user_name varchar(255), - action_type varchar(255), - action_data varchar(1000000), - action_status varchar(255), - action_failure_details varchar(1000000) -); - -CREATE TABLE IF NOT EXISTS attribute_kv ( - entity_type varchar(255), - entity_id uuid, - attribute_type varchar(255), - attribute_key varchar(255), - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, - last_update_ts bigint, - CONSTRAINT attribute_kv_pkey PRIMARY KEY (entity_type, entity_id, attribute_type, attribute_key) -); - -CREATE TABLE IF NOT EXISTS component_descriptor ( - id uuid NOT NULL CONSTRAINT component_descriptor_pkey PRIMARY KEY, - created_time bigint NOT NULL, - actions varchar(255), - clazz varchar UNIQUE, - configuration_descriptor varchar, - name varchar(255), - scope varchar(255), - search_text varchar(255), - type varchar(255) -); - -CREATE TABLE IF NOT EXISTS customer ( - id uuid NOT NULL CONSTRAINT customer_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - address varchar, - address2 varchar, - city varchar(255), - country varchar(255), - email varchar(255), - phone varchar(255), - search_text varchar(255), - state varchar(255), - tenant_id uuid, - title varchar(255), - zip varchar(255) -); - -CREATE TABLE IF NOT EXISTS dashboard ( - id uuid NOT NULL CONSTRAINT dashboard_pkey PRIMARY KEY, - created_time bigint NOT NULL, - configuration varchar, - assigned_customers varchar(1000000), - search_text varchar(255), - tenant_id uuid, - title varchar(255), - mobile_hide boolean DEFAULT false, - mobile_order int, - image varchar(1000000) -); - -CREATE TABLE IF NOT EXISTS rule_chain ( - id uuid NOT NULL CONSTRAINT rule_chain_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - configuration varchar(10000000), - name varchar(255), - type varchar(255), - first_rule_node_id uuid, - root boolean, - debug_mode boolean, - search_text varchar(255), - tenant_id uuid -); - -CREATE TABLE IF NOT EXISTS rule_node ( - id uuid NOT NULL CONSTRAINT rule_node_pkey PRIMARY KEY, - created_time bigint NOT NULL, - rule_chain_id uuid, - additional_info varchar, - configuration varchar(10000000), - type varchar(255), - name varchar(255), - debug_mode boolean, - search_text varchar(255) -); - -CREATE TABLE IF NOT EXISTS rule_node_state ( - id uuid NOT NULL CONSTRAINT rule_node_state_pkey PRIMARY KEY, - created_time bigint NOT NULL, - rule_node_id uuid NOT NULL, - entity_type varchar(32) NOT NULL, - entity_id uuid NOT NULL, - state_data varchar(16384) NOT NULL, - CONSTRAINT rule_node_state_unq_key UNIQUE (rule_node_id, entity_id), - CONSTRAINT fk_rule_node_state_node_id FOREIGN KEY (rule_node_id) REFERENCES rule_node(id) ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS ota_package ( - id uuid NOT NULL CONSTRAINT ota_package_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid NOT NULL, - device_profile_id uuid , - type varchar(32) NOT NULL, - title varchar(255) NOT NULL, - version varchar(255) NOT NULL, - tag varchar(255), - url varchar(255), - file_name varchar(255), - content_type varchar(255), - checksum_algorithm varchar(32), - checksum varchar(1020), - data oid, - data_size bigint, - additional_info varchar, - search_text varchar(255), - CONSTRAINT ota_package_tenant_title_version_unq_key UNIQUE (tenant_id, title, version) -); - -CREATE TABLE IF NOT EXISTS device_profile ( - id uuid NOT NULL CONSTRAINT device_profile_pkey PRIMARY KEY, - created_time bigint NOT NULL, - name varchar(255), - type varchar(255), - image varchar(1000000), - transport_type varchar(255), - provision_type varchar(255), - profile_data jsonb, - description varchar, - search_text varchar(255), - is_default boolean, - tenant_id uuid, - firmware_id uuid, - software_id uuid, - default_rule_chain_id uuid, - default_dashboard_id uuid, - default_queue_name varchar(255), - provision_device_key varchar, - CONSTRAINT device_profile_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT device_provision_key_unq_key UNIQUE (provision_device_key), - CONSTRAINT fk_default_rule_chain_device_profile FOREIGN KEY (default_rule_chain_id) REFERENCES rule_chain(id), - CONSTRAINT fk_default_dashboard_device_profile FOREIGN KEY (default_dashboard_id) REFERENCES dashboard(id), - CONSTRAINT fk_firmware_device_profile FOREIGN KEY (firmware_id) REFERENCES ota_package(id), - CONSTRAINT fk_software_device_profile FOREIGN KEY (software_id) REFERENCES ota_package(id) -); - -ALTER TABLE ota_package - ADD CONSTRAINT fk_device_profile_ota_package - FOREIGN KEY (device_profile_id) REFERENCES device_profile (id) - ON DELETE CASCADE; - --- We will use one-to-many relation in the first release and extend this feature in case of user requests --- CREATE TABLE IF NOT EXISTS device_profile_firmware ( --- device_profile_id uuid NOT NULL, --- firmware_id uuid NOT NULL, --- CONSTRAINT device_profile_firmware_unq_key UNIQUE (device_profile_id, firmware_id), --- CONSTRAINT fk_device_profile_firmware_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id) ON DELETE CASCADE, --- CONSTRAINT fk_device_profile_firmware_firmware FOREIGN KEY (firmware_id) REFERENCES firmware(id) ON DELETE CASCADE, --- ); - -CREATE TABLE IF NOT EXISTS device ( - id uuid NOT NULL CONSTRAINT device_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - device_profile_id uuid NOT NULL, - device_data jsonb, - type varchar(255), - name varchar(255), - label varchar(255), - search_text varchar(255), - tenant_id uuid, - firmware_id uuid, - software_id uuid, - CONSTRAINT device_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT fk_device_profile FOREIGN KEY (device_profile_id) REFERENCES device_profile(id), - CONSTRAINT fk_firmware_device FOREIGN KEY (firmware_id) REFERENCES ota_package(id), - CONSTRAINT fk_software_device FOREIGN KEY (software_id) REFERENCES ota_package(id) -); - -CREATE TABLE IF NOT EXISTS device_credentials ( - id uuid NOT NULL CONSTRAINT device_credentials_pkey PRIMARY KEY, - created_time bigint NOT NULL, - credentials_id varchar, - credentials_type varchar(255), - credentials_value varchar, - device_id uuid, - CONSTRAINT device_credentials_id_unq_key UNIQUE (credentials_id), - CONSTRAINT device_credentials_device_id_unq_key UNIQUE (device_id) -); - -CREATE TABLE IF NOT EXISTS event ( - id uuid NOT NULL CONSTRAINT event_pkey PRIMARY KEY, - created_time bigint NOT NULL, - body varchar(10000000), - entity_id uuid, - entity_type varchar(255), - event_type varchar(255), - event_uid varchar(255), - tenant_id uuid, - ts bigint NOT NULL, - CONSTRAINT event_unq_key UNIQUE (tenant_id, entity_type, entity_id, event_type, event_uid) -); - -CREATE TABLE IF NOT EXISTS relation ( - from_id uuid, - from_type varchar(255), - to_id uuid, - to_type varchar(255), - relation_type_group varchar(255), - relation_type varchar(255), - additional_info varchar, - CONSTRAINT relation_pkey PRIMARY KEY (from_id, from_type, relation_type_group, relation_type, to_id, to_type) -); --- ) PARTITION BY LIST (relation_type_group); --- --- CREATE TABLE other_relations PARTITION OF relation DEFAULT; --- CREATE TABLE common_relations PARTITION OF relation FOR VALUES IN ('COMMON'); --- CREATE TABLE alarm_relations PARTITION OF relation FOR VALUES IN ('ALARM'); --- CREATE TABLE dashboard_relations PARTITION OF relation FOR VALUES IN ('DASHBOARD'); --- CREATE TABLE rule_relations PARTITION OF relation FOR VALUES IN ('RULE_CHAIN', 'RULE_NODE'); - -CREATE TABLE IF NOT EXISTS tb_user ( - id uuid NOT NULL CONSTRAINT tb_user_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - authority varchar(255), - customer_id uuid, - email varchar(255) UNIQUE, - first_name varchar(255), - last_name varchar(255), - search_text varchar(255), - tenant_id uuid -); - -CREATE TABLE IF NOT EXISTS tenant_profile ( - id uuid NOT NULL CONSTRAINT tenant_profile_pkey PRIMARY KEY, - created_time bigint NOT NULL, - name varchar(255), - profile_data jsonb, - description varchar, - search_text varchar(255), - is_default boolean, - isolated_tb_core boolean, - isolated_tb_rule_engine boolean, - CONSTRAINT tenant_profile_name_unq_key UNIQUE (name) -); - -CREATE TABLE IF NOT EXISTS tenant ( - id uuid NOT NULL CONSTRAINT tenant_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - tenant_profile_id uuid NOT NULL, - address varchar, - address2 varchar, - city varchar(255), - country varchar(255), - email varchar(255), - phone varchar(255), - region varchar(255), - search_text varchar(255), - state varchar(255), - title varchar(255), - zip varchar(255), - CONSTRAINT fk_tenant_profile FOREIGN KEY (tenant_profile_id) REFERENCES tenant_profile(id) -); - -CREATE TABLE IF NOT EXISTS user_credentials ( - id uuid NOT NULL CONSTRAINT user_credentials_pkey PRIMARY KEY, - created_time bigint NOT NULL, - activate_token varchar(255) UNIQUE, - enabled boolean, - password varchar(255), - reset_token varchar(255) UNIQUE, - user_id uuid UNIQUE -); - -CREATE TABLE IF NOT EXISTS widget_type ( - id uuid NOT NULL CONSTRAINT widget_type_pkey PRIMARY KEY, - created_time bigint NOT NULL, - alias varchar(255), - bundle_alias varchar(255), - descriptor varchar(1000000), - name varchar(255), - tenant_id uuid, - image varchar(1000000), - description varchar(255) -); - -CREATE TABLE IF NOT EXISTS widgets_bundle ( - id uuid NOT NULL CONSTRAINT widgets_bundle_pkey PRIMARY KEY, - created_time bigint NOT NULL, - alias varchar(255), - search_text varchar(255), - tenant_id uuid, - title varchar(255), - image varchar(1000000), - description varchar(255) -); - -CREATE TABLE IF NOT EXISTS entity_view ( - id uuid NOT NULL CONSTRAINT entity_view_pkey PRIMARY KEY, - created_time bigint NOT NULL, - entity_id uuid, - entity_type varchar(255), - tenant_id uuid, - customer_id uuid, - type varchar(255), - name varchar(255), - keys varchar(10000000), - start_ts bigint, - end_ts bigint, - search_text varchar(255), - additional_info varchar -); - -CREATE TABLE IF NOT EXISTS ts_kv_latest -( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, - CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); - -CREATE TABLE IF NOT EXISTS ts_kv_dictionary -( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) -); - -CREATE TABLE IF NOT EXISTS oauth2_params ( - id uuid NOT NULL CONSTRAINT oauth2_params_pkey PRIMARY KEY, - enabled boolean, - tenant_id uuid, - created_time bigint NOT NULL -); - -CREATE TABLE IF NOT EXISTS oauth2_registration ( - id uuid NOT NULL CONSTRAINT oauth2_registration_pkey PRIMARY KEY, - oauth2_params_id uuid NOT NULL, - created_time bigint NOT NULL, - additional_info varchar, - client_id varchar(255), - client_secret varchar(2048), - authorization_uri varchar(255), - token_uri varchar(255), - scope varchar(255), - platforms varchar(255), - user_info_uri varchar(255), - user_name_attribute_name varchar(255), - jwk_set_uri varchar(255), - client_authentication_method varchar(255), - login_button_label varchar(255), - login_button_icon varchar(255), - allow_user_creation boolean, - activate_user boolean, - type varchar(31), - basic_email_attribute_key varchar(31), - basic_first_name_attribute_key varchar(31), - basic_last_name_attribute_key varchar(31), - basic_tenant_name_strategy varchar(31), - basic_tenant_name_pattern varchar(255), - basic_customer_name_pattern varchar(255), - basic_default_dashboard_name varchar(255), - basic_always_full_screen boolean, - custom_url varchar(255), - custom_username varchar(255), - custom_password varchar(255), - custom_send_token boolean, - CONSTRAINT fk_registration_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE -); - -CREATE TABLE IF NOT EXISTS oauth2_domain ( - id uuid NOT NULL CONSTRAINT oauth2_domain_pkey PRIMARY KEY, - oauth2_params_id uuid NOT NULL, - created_time bigint NOT NULL, - domain_name varchar(255), - domain_scheme varchar(31), - CONSTRAINT fk_domain_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE, - CONSTRAINT oauth2_domain_unq_key UNIQUE (oauth2_params_id, domain_name, domain_scheme) -); - -CREATE TABLE IF NOT EXISTS oauth2_mobile ( - id uuid NOT NULL CONSTRAINT oauth2_mobile_pkey PRIMARY KEY, - oauth2_params_id uuid NOT NULL, - created_time bigint NOT NULL, - pkg_name varchar(255), - app_secret varchar(2048), - CONSTRAINT fk_mobile_oauth2_params FOREIGN KEY (oauth2_params_id) REFERENCES oauth2_params(id) ON DELETE CASCADE, - CONSTRAINT oauth2_mobile_unq_key UNIQUE (oauth2_params_id, pkg_name) -); - -CREATE TABLE IF NOT EXISTS oauth2_client_registration_template ( - id uuid NOT NULL CONSTRAINT oauth2_client_registration_template_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - provider_id varchar(255), - authorization_uri varchar(255), - token_uri varchar(255), - scope varchar(255), - user_info_uri varchar(255), - user_name_attribute_name varchar(255), - jwk_set_uri varchar(255), - client_authentication_method varchar(255), - type varchar(31), - basic_email_attribute_key varchar(31), - basic_first_name_attribute_key varchar(31), - basic_last_name_attribute_key varchar(31), - basic_tenant_name_strategy varchar(31), - basic_tenant_name_pattern varchar(255), - basic_customer_name_pattern varchar(255), - basic_default_dashboard_name varchar(255), - basic_always_full_screen boolean, - comment varchar, - login_button_icon varchar(255), - login_button_label varchar(255), - help_link varchar(255), - CONSTRAINT oauth2_template_provider_id_unq_key UNIQUE (provider_id) -); - --- Deprecated -CREATE TABLE IF NOT EXISTS oauth2_client_registration_info ( - id uuid NOT NULL CONSTRAINT oauth2_client_registration_info_pkey PRIMARY KEY, - enabled boolean, - created_time bigint NOT NULL, - additional_info varchar, - client_id varchar(255), - client_secret varchar(255), - authorization_uri varchar(255), - token_uri varchar(255), - scope varchar(255), - user_info_uri varchar(255), - user_name_attribute_name varchar(255), - jwk_set_uri varchar(255), - client_authentication_method varchar(255), - login_button_label varchar(255), - login_button_icon varchar(255), - allow_user_creation boolean, - activate_user boolean, - type varchar(31), - basic_email_attribute_key varchar(31), - basic_first_name_attribute_key varchar(31), - basic_last_name_attribute_key varchar(31), - basic_tenant_name_strategy varchar(31), - basic_tenant_name_pattern varchar(255), - basic_customer_name_pattern varchar(255), - basic_default_dashboard_name varchar(255), - basic_always_full_screen boolean, - custom_url varchar(255), - custom_username varchar(255), - custom_password varchar(255), - custom_send_token boolean -); - --- Deprecated -CREATE TABLE IF NOT EXISTS oauth2_client_registration ( - id uuid NOT NULL CONSTRAINT oauth2_client_registration_pkey PRIMARY KEY, - created_time bigint NOT NULL, - domain_name varchar(255), - domain_scheme varchar(31), - client_registration_info_id uuid -); - -CREATE TABLE IF NOT EXISTS api_usage_state ( - id uuid NOT NULL CONSTRAINT usage_record_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid, - entity_type varchar(32), - entity_id uuid, - transport varchar(32), - db_storage varchar(32), - re_exec varchar(32), - js_exec varchar(32), - email_exec varchar(32), - sms_exec varchar(32), - alarm_exec varchar(32), - CONSTRAINT api_usage_state_unq_key UNIQUE (tenant_id, entity_id) -); - -CREATE TABLE IF NOT EXISTS resource ( - id uuid NOT NULL CONSTRAINT resource_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid NOT NULL, - title varchar(255) NOT NULL, - resource_type varchar(32) NOT NULL, - resource_key varchar(255) NOT NULL, - search_text varchar(255), - file_name varchar(255) NOT NULL, - data varchar, - CONSTRAINT resource_unq_key UNIQUE (tenant_id, resource_type, resource_key) -); - -CREATE TABLE IF NOT EXISTS edge ( - id uuid NOT NULL CONSTRAINT edge_pkey PRIMARY KEY, - created_time bigint NOT NULL, - additional_info varchar, - customer_id uuid, - root_rule_chain_id uuid, - type varchar(255), - name varchar(255), - label varchar(255), - routing_key varchar(255), - secret varchar(255), - search_text varchar(255), - tenant_id uuid, - CONSTRAINT edge_name_unq_key UNIQUE (tenant_id, name), - CONSTRAINT edge_routing_key_unq_key UNIQUE (routing_key) -); - -CREATE TABLE IF NOT EXISTS edge_event ( - id uuid NOT NULL CONSTRAINT edge_event_pkey PRIMARY KEY, - created_time bigint NOT NULL, - edge_id uuid, - edge_event_type varchar(255), - edge_event_uid varchar(255), - entity_id uuid, - edge_event_action varchar(255), - body varchar(10000000), - tenant_id uuid, - ts bigint NOT NULL -); - -CREATE TABLE IF NOT EXISTS rpc ( - id uuid NOT NULL CONSTRAINT rpc_pkey PRIMARY KEY, - created_time bigint NOT NULL, - tenant_id uuid NOT NULL, - device_id uuid NOT NULL, - expiration_time bigint NOT NULL, - request varchar(10000000) NOT NULL, - response varchar(10000000), - additional_info varchar(10000000), - status varchar(255) NOT NULL -); - -CREATE OR REPLACE PROCEDURE cleanup_events_by_ttl( - IN regular_events_start_ts bigint, - IN regular_events_end_ts bigint, - IN debug_events_start_ts bigint, - IN debug_events_end_ts bigint, - INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_deleted_count bigint DEFAULT 0; - debug_ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF regular_events_start_ts > 0 AND regular_events_end_ts > 0 THEN - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE id in (SELECT id from event WHERE ts > %L::bigint AND ts < %L::bigint AND ' || - '(event_type != %L::varchar AND event_type != %L::varchar)) RETURNING *) ' || - 'SELECT count(*) FROM deleted', regular_events_start_ts, regular_events_end_ts, - 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into ttl_deleted_count; - END IF; - IF debug_events_start_ts > 0 AND debug_events_end_ts > 0 THEN - EXECUTE format( - 'WITH deleted AS (DELETE FROM event WHERE id in (SELECT id from event WHERE ts > %L::bigint AND ts < %L::bigint AND ' || - '(event_type = %L::varchar OR event_type = %L::varchar)) RETURNING *) ' || - 'SELECT count(*) FROM deleted', debug_events_start_ts, debug_events_end_ts, - 'DEBUG_RULE_NODE', 'DEBUG_RULE_CHAIN') into debug_ttl_deleted_count; - END IF; - RAISE NOTICE 'Events removed by ttl: %', ttl_deleted_count; - RAISE NOTICE 'Debug Events removed by ttl: %', debug_ttl_deleted_count; - deleted := ttl_deleted_count + debug_ttl_deleted_count; -END -$$; - -CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS -$$ -BEGIN - uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || - '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); -END; -$$ LANGUAGE plpgsql; - - -CREATE OR REPLACE PROCEDURE cleanup_edge_events_by_ttl(IN ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - ttl_ts bigint; - ttl_deleted_count bigint DEFAULT 0; -BEGIN - IF ttl > 0 THEN - ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - ttl::bigint * 1000)::bigint; - EXECUTE format( - 'WITH deleted AS (DELETE FROM edge_event WHERE ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', ttl_ts) into ttl_deleted_count; - END IF; - RAISE NOTICE 'Edge events removed by ttl: %', ttl_deleted_count; - deleted := ttl_deleted_count; -END -$$; diff --git a/application/src/main/data/sql/schema-timescale.sql b/application/src/main/data/sql/schema-timescale.sql deleted file mode 100644 index 81b71f1a03..0000000000 --- a/application/src/main/data/sql/schema-timescale.sql +++ /dev/null @@ -1,157 +0,0 @@ --- --- Copyright © 2016-2022 The Thingsboard Authors --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE; - -CREATE TABLE IF NOT EXISTS ts_kv ( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, - CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) -); - -CREATE TABLE IF NOT EXISTS ts_kv_dictionary ( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) -); - -CREATE TABLE IF NOT EXISTS ts_kv_latest ( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, - CONSTRAINT ts_kv_latest_pkey PRIMARY KEY (entity_id, key) -); - -CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS -$$ -BEGIN - uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || - '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, - OUT deleted bigint) AS -$$ -BEGIN - EXECUTE format( - 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT device.id as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', - tenant_id, customer_id, ttl) into deleted; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, - OUT deleted bigint) AS -$$ -BEGIN - EXECUTE format( - 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT asset.id as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', - tenant_id, customer_id, ttl) into deleted; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, - OUT deleted bigint) AS -$$ -BEGIN - EXECUTE format( - 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT customer.id as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', - tenant_id, customer_id, ttl) into deleted; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, - IN system_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - tenant_cursor CURSOR FOR select tenant.id as tenant_id - from tenant; - tenant_id_record uuid; - customer_id_record uuid; - tenant_ttl bigint; - customer_ttl bigint; - deleted_for_entities bigint; - tenant_ttl_ts bigint; - customer_ttl_ts bigint; -BEGIN - OPEN tenant_cursor; - FETCH tenant_cursor INTO tenant_id_record; - WHILE FOUND - LOOP - EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', - tenant_id_record, 'TTL') INTO tenant_ttl; - if tenant_ttl IS NULL THEN - tenant_ttl := system_ttl; - END IF; - IF tenant_ttl > 0 THEN - tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; - deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; - deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; - END IF; - FOR customer_id_record IN - SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record - LOOP - EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', - customer_id_record, 'TTL') INTO customer_ttl; - IF customer_ttl IS NULL THEN - customer_ttl_ts := tenant_ttl_ts; - ELSE - IF customer_ttl > 0 THEN - customer_ttl_ts := - (EXTRACT(EPOCH FROM current_timestamp) * 1000 - - customer_ttl::bigint * 1000)::bigint; - END IF; - END IF; - IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN - deleted_for_entities := - delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; - deleted_for_entities := - delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; - deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, - customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; - END IF; - END LOOP; - FETCH tenant_cursor INTO tenant_id_record; - END LOOP; -END -$$; diff --git a/application/src/main/data/sql/schema-ts-psql.sql b/application/src/main/data/sql/schema-ts-psql.sql deleted file mode 100644 index c95ac4cf0f..0000000000 --- a/application/src/main/data/sql/schema-ts-psql.sql +++ /dev/null @@ -1,339 +0,0 @@ --- --- Copyright © 2016-2022 The Thingsboard Authors --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - -CREATE TABLE IF NOT EXISTS ts_kv -( - entity_id uuid NOT NULL, - key int NOT NULL, - ts bigint NOT NULL, - bool_v boolean, - str_v varchar(10000000), - long_v bigint, - dbl_v double precision, - json_v json, - CONSTRAINT ts_kv_pkey PRIMARY KEY (entity_id, key, ts) -) PARTITION BY RANGE (ts); - -CREATE TABLE IF NOT EXISTS ts_kv_dictionary -( - key varchar(255) NOT NULL, - key_id serial UNIQUE, - CONSTRAINT ts_key_id_pkey PRIMARY KEY (key) -); - -CREATE OR REPLACE PROCEDURE drop_partitions_by_max_ttl(IN partition_type varchar, IN system_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - max_tenant_ttl bigint; - max_customer_ttl bigint; - max_ttl bigint; - date timestamp; - partition_by_max_ttl_date varchar; - partition_by_max_ttl_month varchar; - partition_by_max_ttl_day varchar; - partition_by_max_ttl_year varchar; - partition varchar; - partition_year integer; - partition_month integer; - partition_day integer; - -BEGIN - SELECT max(attribute_kv.long_v) - FROM tenant - INNER JOIN attribute_kv ON tenant.id = attribute_kv.entity_id - WHERE attribute_kv.attribute_key = 'TTL' - into max_tenant_ttl; - SELECT max(attribute_kv.long_v) - FROM customer - INNER JOIN attribute_kv ON customer.id = attribute_kv.entity_id - WHERE attribute_kv.attribute_key = 'TTL' - into max_customer_ttl; - max_ttl := GREATEST(system_ttl, max_customer_ttl, max_tenant_ttl); - if max_ttl IS NOT NULL AND max_ttl > 0 THEN - date := to_timestamp(EXTRACT(EPOCH FROM current_timestamp) - max_ttl); - partition_by_max_ttl_date := get_partition_by_max_ttl_date(partition_type, date); - RAISE NOTICE 'Date by max ttl: %', date; - RAISE NOTICE 'Partition by max ttl: %', partition_by_max_ttl_date; - IF partition_by_max_ttl_date IS NOT NULL THEN - CASE - WHEN partition_type = 'DAYS' THEN - partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); - partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); - partition_by_max_ttl_day := SPLIT_PART(partition_by_max_ttl_date, '_', 5); - WHEN partition_type = 'MONTHS' THEN - partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); - partition_by_max_ttl_month := SPLIT_PART(partition_by_max_ttl_date, '_', 4); - ELSE - partition_by_max_ttl_year := SPLIT_PART(partition_by_max_ttl_date, '_', 3); - END CASE; - IF partition_by_max_ttl_year IS NULL THEN - RAISE NOTICE 'Failed to remove partitions by max ttl date due to partition_by_max_ttl_year is null!'; - ELSE - IF partition_type = 'YEARS' THEN - FOR partition IN SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename like 'ts_kv_' || '%' - AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' - AND tablename != 'ts_kv_indefinite' - AND tablename != partition_by_max_ttl_date - LOOP - partition_year := SPLIT_PART(partition, '_', 3)::integer; - IF partition_year < partition_by_max_ttl_year::integer THEN - RAISE NOTICE 'Partition to delete by max ttl: %', partition; - EXECUTE format('DROP TABLE IF EXISTS %I', partition); - deleted := deleted + 1; - END IF; - END LOOP; - ELSE - IF partition_type = 'MONTHS' THEN - IF partition_by_max_ttl_month IS NULL THEN - RAISE NOTICE 'Failed to remove months partitions by max ttl date due to partition_by_max_ttl_month is null!'; - ELSE - FOR partition IN SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename like 'ts_kv_' || '%' - AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' - AND tablename != 'ts_kv_indefinite' - AND tablename != partition_by_max_ttl_date - LOOP - partition_year := SPLIT_PART(partition, '_', 3)::integer; - IF partition_year > partition_by_max_ttl_year::integer THEN - RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; - CONTINUE; - ELSE - IF partition_year < partition_by_max_ttl_year::integer THEN - RAISE NOTICE 'Partition to delete by max ttl: %', partition; - EXECUTE format('DROP TABLE IF EXISTS %I', partition); - deleted := deleted + 1; - ELSE - partition_month := SPLIT_PART(partition, '_', 4)::integer; - IF partition_year = partition_by_max_ttl_year::integer THEN - IF partition_month >= partition_by_max_ttl_month::integer THEN - RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; - CONTINUE; - ELSE - RAISE NOTICE 'Partition to delete by max ttl: %', partition; - EXECUTE format('DROP TABLE IF EXISTS %I', partition); - deleted := deleted + 1; - END IF; - END IF; - END IF; - END IF; - END LOOP; - END IF; - ELSE - IF partition_type = 'DAYS' THEN - IF partition_by_max_ttl_month IS NULL THEN - RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_month is null!'; - ELSE - IF partition_by_max_ttl_day IS NULL THEN - RAISE NOTICE 'Failed to remove days partitions by max ttl date due to partition_by_max_ttl_day is null!'; - ELSE - FOR partition IN SELECT tablename - FROM pg_tables - WHERE schemaname = 'public' - AND tablename like 'ts_kv_' || '%' - AND tablename != 'ts_kv_latest' - AND tablename != 'ts_kv_dictionary' - AND tablename != 'ts_kv_indefinite' - AND tablename != partition_by_max_ttl_date - LOOP - partition_year := SPLIT_PART(partition, '_', 3)::integer; - IF partition_year > partition_by_max_ttl_year::integer THEN - RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; - CONTINUE; - ELSE - IF partition_year < partition_by_max_ttl_year::integer THEN - RAISE NOTICE 'Partition to delete by max ttl: %', partition; - EXECUTE format('DROP TABLE IF EXISTS %I', partition); - deleted := deleted + 1; - ELSE - partition_month := SPLIT_PART(partition, '_', 4)::integer; - IF partition_month > partition_by_max_ttl_month::integer THEN - RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; - CONTINUE; - ELSE - IF partition_month < partition_by_max_ttl_month::integer THEN - RAISE NOTICE 'Partition to delete by max ttl: %', partition; - EXECUTE format('DROP TABLE IF EXISTS %I', partition); - deleted := deleted + 1; - ELSE - partition_day := SPLIT_PART(partition, '_', 5)::integer; - IF partition_day >= partition_by_max_ttl_day::integer THEN - RAISE NOTICE 'Skip iteration! Partition: % is valid!', partition; - CONTINUE; - ELSE - IF partition_day < partition_by_max_ttl_day::integer THEN - RAISE NOTICE 'Partition to delete by max ttl: %', partition; - EXECUTE format('DROP TABLE IF EXISTS %I', partition); - deleted := deleted + 1; - END IF; - END IF; - END IF; - END IF; - END IF; - END IF; - END LOOP; - END IF; - END IF; - END IF; - END IF; - END IF; - END IF; - END IF; - END IF; -END -$$; - -CREATE OR REPLACE FUNCTION get_partition_by_max_ttl_date(IN partition_type varchar, IN date timestamp, OUT partition varchar) AS -$$ -BEGIN - CASE - WHEN partition_type = 'DAYS' THEN - partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM') || '_' || to_char(date, 'dd'); - WHEN partition_type = 'MONTHS' THEN - partition := 'ts_kv_' || to_char(date, 'yyyy') || '_' || to_char(date, 'MM'); - WHEN partition_type = 'YEARS' THEN - partition := 'ts_kv_' || to_char(date, 'yyyy'); - ELSE - partition := NULL; - END CASE; - IF partition IS NOT NULL THEN - IF NOT EXISTS(SELECT - FROM pg_tables - WHERE schemaname = 'public' - AND tablename = partition) THEN - partition := NULL; - RAISE NOTICE 'Failed to found partition by ttl'; - END IF; - END IF; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION to_uuid(IN entity_id varchar, OUT uuid_id uuid) AS -$$ -BEGIN - uuid_id := substring(entity_id, 8, 8) || '-' || substring(entity_id, 4, 4) || '-1' || substring(entity_id, 1, 3) || - '-' || substring(entity_id, 16, 4) || '-' || substring(entity_id, 20, 12); -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_device_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, - OUT deleted bigint) AS -$$ -BEGIN - EXECUTE format( - 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT device.id as entity_id FROM device WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', - tenant_id, customer_id, ttl) into deleted; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_asset_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, - OUT deleted bigint) AS -$$ -BEGIN - EXECUTE format( - 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT asset.id as entity_id FROM asset WHERE tenant_id = %L and customer_id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', - tenant_id, customer_id, ttl) into deleted; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION delete_customer_records_from_ts_kv(tenant_id uuid, customer_id uuid, ttl bigint, - OUT deleted bigint) AS -$$ -BEGIN - EXECUTE format( - 'WITH deleted AS (DELETE FROM ts_kv WHERE entity_id IN (SELECT customer.id as entity_id FROM customer WHERE tenant_id = %L and id = %L) AND ts < %L::bigint RETURNING *) SELECT count(*) FROM deleted', - tenant_id, customer_id, ttl) into deleted; -END; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE PROCEDURE cleanup_timeseries_by_ttl(IN null_uuid uuid, - IN system_ttl bigint, INOUT deleted bigint) - LANGUAGE plpgsql AS -$$ -DECLARE - tenant_cursor CURSOR FOR select tenant.id as tenant_id - from tenant; - tenant_id_record uuid; - customer_id_record uuid; - tenant_ttl bigint; - customer_ttl bigint; - deleted_for_entities bigint; - tenant_ttl_ts bigint; - customer_ttl_ts bigint; -BEGIN - OPEN tenant_cursor; - FETCH tenant_cursor INTO tenant_id_record; - WHILE FOUND - LOOP - EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', - tenant_id_record, 'TTL') INTO tenant_ttl; - if tenant_ttl IS NULL THEN - tenant_ttl := system_ttl; - END IF; - IF tenant_ttl > 0 THEN - tenant_ttl_ts := (EXTRACT(EPOCH FROM current_timestamp) * 1000 - tenant_ttl::bigint * 1000)::bigint; - deleted_for_entities := delete_device_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for devices where tenant_id = %', deleted_for_entities, tenant_id_record; - deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, null_uuid, tenant_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for assets where tenant_id = %', deleted_for_entities, tenant_id_record; - END IF; - FOR customer_id_record IN - SELECT customer.id AS customer_id FROM customer WHERE customer.tenant_id = tenant_id_record - LOOP - EXECUTE format( - 'select attribute_kv.long_v from attribute_kv where attribute_kv.entity_id = %L and attribute_kv.attribute_key = %L', - customer_id_record, 'TTL') INTO customer_ttl; - IF customer_ttl IS NULL THEN - customer_ttl_ts := tenant_ttl_ts; - ELSE - IF customer_ttl > 0 THEN - customer_ttl_ts := - (EXTRACT(EPOCH FROM current_timestamp) * 1000 - - customer_ttl::bigint * 1000)::bigint; - END IF; - END IF; - IF customer_ttl_ts IS NOT NULL AND customer_ttl_ts > 0 THEN - deleted_for_entities := - delete_customer_records_from_ts_kv(tenant_id_record, customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for customer with id = % where tenant_id = %', deleted_for_entities, customer_id_record, tenant_id_record; - deleted_for_entities := - delete_device_records_from_ts_kv(tenant_id_record, customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for devices where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; - deleted_for_entities := delete_asset_records_from_ts_kv(tenant_id_record, - customer_id_record, - customer_ttl_ts); - deleted := deleted + deleted_for_entities; - RAISE NOTICE '% telemetry removed for assets where tenant_id = % and customer_id = %', deleted_for_entities, tenant_id_record, customer_id_record; - END IF; - END LOOP; - FETCH tenant_cursor INTO tenant_id_record; - END LOOP; -END -$$; diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index c1aadc238b..da758f5f27 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,17 +20,11 @@ import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.hibernate.engine.jdbc.BlobProxy; import org.springframework.core.io.FileSystemResource; -import org.springframework.http.*; +import org.springframework.http.HttpHeaders; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; -import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.OtaPackage; @@ -50,27 +44,9 @@ import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import javax.transaction.Transactional; -import java.io.InputStream; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.sql.Blob; +import java.io.IOException; -import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_ID_PARAM_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES; -import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_ID_PARAM_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_INFO_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_SORT_PROPERTY_ALLOWABLE_VALUES; -import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_TEXT_SEARCH_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES; -import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH; -import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; -import static org.thingsboard.server.controller.ControllerConstants.UUID_WIKI_LINK; +import static org.thingsboard.server.controller.ControllerConstants.*; @Slf4j @RestController @@ -89,7 +65,7 @@ public class OtaPackageController extends BaseController { @ResponseBody @Transactional public ResponseEntity downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) - @PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException { + @PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException { checkParameter(OTA_PACKAGE_ID, strOtaPackageId); try { OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId)); @@ -205,7 +181,7 @@ public class OtaPackageController extends BaseController { } } - private OtaPackage saveOtaPackageWithData(OtaPackageId otaPackageId, MultipartFile file, String checksum, ChecksumAlgorithm checksumAlgorithm) throws ThingsboardException { + private OtaPackage saveOtaPackageWithData(OtaPackageId otaPackageId, MultipartFile file, String checksum, ChecksumAlgorithm checksumAlgorithm) throws ThingsboardException, IOException { OtaPackageInfo info = checkOtaPackageInfoId(otaPackageId, Operation.READ); OtaPackage otaPackage = new OtaPackage(otaPackageId); otaPackage.setCreatedTime(info.getCreatedTime()); @@ -218,17 +194,11 @@ public class OtaPackageController extends BaseController { otaPackage.setAdditionalInfo(info.getAdditionalInfo()); otaPackage.setChecksumAlgorithm(checksumAlgorithm); otaPackage.setChecksum(checksum); - try(InputStream inputStream = file.getInputStream()) { - otaPackage.setFileName(file.getOriginalFilename()); - otaPackage.setContentType(file.getContentType()); - otaPackage.setDataSize(file.getSize()); - Blob blob = BlobProxy.generateProxy(inputStream, file.getSize()); - otaPackage.setData(blob); - return otaPackageService.saveOtaPackage(otaPackage); - } catch (Exception e){ - log.error("Failed to save ota package {} with data {}",otaPackageId, file.getName(), e); - return null; - } + otaPackage.setFileName(file.getOriginalFilename()); + otaPackage.setContentType(file.getContentType()); + otaPackage.setDataSize(file.getSize()); + otaPackage.setData(file.getInputStream()); + return otaPackageService.saveOtaPackage(otaPackage); } @ApiOperation(value = "Get OTA Package Infos (getOtaPackages)", diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index 821ecd2b35..bc17d0fc93 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,9 +22,9 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; +import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.audit.ActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; @@ -37,7 +37,6 @@ import org.thingsboard.server.common.msg.TbMsgDataType; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.cluster.TbClusterService; import java.util.List; import java.util.Map; @@ -155,15 +154,9 @@ public class EntityActionService { } ObjectNode entityNode; if (entity != null) { - if (entity instanceof OtaPackage) { - OtaPackage otaPackage = (OtaPackage) entity; - otaPackage.setData(null); - entityNode = json.valueToTree(otaPackage); //todo - } else { - entityNode = json.valueToTree(entity); - if (entityId.getEntityType() == EntityType.DASHBOARD) { - entityNode.put("configuration", ""); - } + entityNode = json.valueToTree(entity); + if (entityId.getEntityType() == EntityType.DASHBOARD) { + entityNode.put("configuration", ""); } } else { entityNode = json.createObjectNode(); @@ -214,8 +207,8 @@ public class EntityActionService { } } - public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, - ActionType actionType, Exception e, Object... additionalInfo) { + public void logEntityAction(User user, I entityId, E entity, CustomerId customerId, + ActionType actionType, Exception e, Object... additionalInfo) { if (customerId == null || customerId.isNullUid()) { customerId = user.getCustomerId(); } diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index ae6e0a6462..b9a7b50576 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -99,6 +99,7 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.resource.TbResourceService; +import java.io.IOException; import java.sql.SQLException; import java.util.Optional; import java.util.UUID; @@ -573,7 +574,6 @@ public class DefaultTransportApiService implements TransportApiService { DeviceProfile deviceProfile = deviceProfileCache.find(device.getDeviceProfileId()); otaPackageId = OtaPackageUtil.getOtaPackageId(deviceProfile, otaPackageType); } - TransportProtos.GetOtaPackageResponseMsg.Builder builder = TransportProtos.GetOtaPackageResponseMsg.newBuilder(); if (otaPackageId == null) { @@ -598,9 +598,9 @@ public class DefaultTransportApiService implements TransportApiService { if (!otaPackageDataCache.has(otaPackageId.toString())) { OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); try { - otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().getBytes(100L, 0));//todo - } catch (SQLException e) { - log.error("ERROR", e); // todo + otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().readAllBytes()); + } catch (IOException e) { + log.error("Failed to cache ota package with id {}",otaPackage.getId(), e); } } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 4c181c0218..79b33785c1 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -505,8 +505,8 @@ spring.resources.chain: content: enabled: "true" -spring.servlet.multipart.max-file-size: "2000MB" -spring.servlet.multipart.max-request-size: "2000MB" +spring.servlet.multipart.max-file-size: "50MB" +spring.servlet.multipart.max-request-size: "50MB" spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true" spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" @@ -526,9 +526,9 @@ spring: database-platform: "${SPRING_JPA_DATABASE_PLATFORM:org.hibernate.dialect.PostgreSQLDialect}" datasource: driverClassName: "${SPRING_DRIVER_CLASS_NAME:org.postgresql.Driver}" - url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/board}" - username: "${SPRING_DATASOURCE_USERNAME:myuser}" - password: "${SPRING_DATASOURCE_PASSWORD:mypass}" + url: "${SPRING_DATASOURCE_URL:jdbc:postgresql://localhost:5432/thingsboard}" + username: "${SPRING_DATASOURCE_USERNAME:postgres}" + password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:16}" @@ -1114,3 +1114,6 @@ management: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). include: '${METRICS_ENDPOINTS_EXPOSE:info}' + +files: + temporary_files_directory: '${TEMPORARY_FILES_DIRECTORY: ${java.io.tmpdir}}' \ No newline at end of file diff --git a/common/cache/pom.xml b/common/cache/pom.xml index c912c4eede..33cd95fe5b 100644 --- a/common/cache/pom.xml +++ b/common/cache/pom.xml @@ -40,14 +40,15 @@ org.thingsboard.common data - - org.thingsboard.common - dao-api - org.springframework.boot spring-boot-autoconfigure + + javax.annotation + javax.annotation-api + 1.3.2 + org.springframework.data spring-data-redis @@ -95,12 +96,6 @@ awaitility test - - org.hibernate - hibernate-core - 5.4.32.Final - test - diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java new file mode 100644 index 0000000000..bbd0056e92 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java @@ -0,0 +1,100 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.ota.files; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.OtaPackageId; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +@Slf4j +@RequiredArgsConstructor +@Component +public class BaseFileCacheService implements FileCacheService { + @Value("${files.temporary_files_directory}/ota/") + private String PATH; + private final static String FILE_NAME_TEMPLATE = "%s.tmp"; + private final TemporaryFileCleaner fileCleaner; + private final ConcurrentMap files = new ConcurrentHashMap<>(); + + + @Override + public File saveDataTemporaryFile(InputStream inputStream) { + File path = new File(PATH); + try { + File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); + FileUtils.copyInputStreamToFile(inputStream, tempFile); + return tempFile; + } catch (IOException e) { + log.error("Failed to create temp file", e); + throw new RuntimeException("Failed to create temp file for input stream"); + } + } + + @Override + public Optional getOtaDataFile(OtaPackageId otaPackageId) { + String fileName = PATH + String.format(FILE_NAME_TEMPLATE,otaPackageId.getId().toString()); + if (exist(fileName)) { + fileCleaner.updateFileUsageStatus(otaPackageId); + return Optional.of(new File(fileName)); + } + return Optional.empty(); + } + + @Override + public File loadToFile(OtaPackageId otaPackageId, InputStream data) { + if (otaPackageId == null || data == null) { + log.error("Received null variables: {}", otaPackageId == null ? "otaPackageId" : "data"); + throw new RuntimeException("Input values can not be null"); + } + files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data)); + String fileName = PATH + String.format(FILE_NAME_TEMPLATE,otaPackageId.getId().toString()); + return new File(fileName); + } + + + private Boolean processFileSaving(OtaPackageId otaPackageId, InputStream data) { + String fileName = PATH + String.format(FILE_NAME_TEMPLATE,otaPackageId.getId().toString()); + saveAsSystemFile(fileName, data); + fileCleaner.updateFileUsageStatus(otaPackageId); + return true; + } + + private void saveAsSystemFile(String fileName, InputStream inputStream) { + try { + File file = new File(fileName); + FileUtils.copyInputStreamToFile(inputStream, file); + } catch (IOException e) { + log.error("Failed to copy stream to system file {}", fileName, e); + throw new RuntimeException("Failed to save file"); + } + } + + private boolean exist(String name) { + File file = new File(name); + return file.exists(); + } +} \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java new file mode 100644 index 0000000000..4575a3502c --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java @@ -0,0 +1,14 @@ +package org.thingsboard.server.cache.ota.files; + +import org.thingsboard.server.common.data.id.OtaPackageId; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Optional; + +public interface FileCacheService { + File loadToFile(OtaPackageId otaPackageId, InputStream data); + File saveDataTemporaryFile(InputStream inputStream); + Optional getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException; +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java similarity index 63% rename from common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java rename to common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java index dc943c83d9..f9a8363376 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java @@ -1,38 +1,44 @@ -package org.thingsboard.server.cache.ota.service; +package org.thingsboard.server.cache.ota.files; -import lombok.extern.apachecommons.CommonsLog; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.OtaPackageId; -import javax.annotation.PostConstruct; import java.io.File; import java.io.IOException; +import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.concurrent.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; @Slf4j @Component public class TemporaryFileCleaner { - @Value("${java.io.tmpdir}/ota/") + @Value("${files.temporary_files_directory}/ota/") private String PATH; private final static long TEMPORARY_FILE_INACTIVITY_TIME = 900_000; private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); - public void updateFileUsageStatus(OtaPackageId otaPackageId){ + public void updateFileUsageStatus(OtaPackageId otaPackageId) { lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); } - @PostConstruct + @EventListener(ApplicationReadyEvent.class) public void createScheduledTaskToClear() { createTempDirectoryIfNotExist(); cleanDirectoryWithTemporaryFiles(); - ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 10, 10, TimeUnit.MINUTES); } private void createTempDirectoryIfNotExist() { @@ -45,16 +51,21 @@ public class TemporaryFileCleaner { private void cleanDirectoryWithTemporaryFiles() { File directory = new File(PATH); if (directory.isDirectory()) { - try { - FileUtils.cleanDirectory(directory); - log.info("Directory with temporary files cleaned"); - } catch (IOException e) { - log.error("Failed to clean directory with temporary files", e); - throw new RuntimeException(e); - } + File[] files = directory.listFiles(); + if (files == null) return; + Arrays.stream(files).forEach( + file -> { + try { + FileUtils.delete(file); + } catch (Exception e) { + log.error("Failed to delete file {}", file.getName(), e); + } + } + ); } } + @Scheduled(fixedDelay = 60000) private void deleteUnusedTemporaryFiles() { long currentTime = System.currentTimeMillis(); List toBeDeleted = lastActivityTimes.entrySet() @@ -73,14 +84,16 @@ public class TemporaryFileCleaner { log.info("Deleted {} unused temporary files", toBeDeleted.size()); } - private void deleteFile(String otaId) { + private synchronized void deleteFile(String otaId) { String fileName = PATH + otaId; File file = new File(fileName); - try { + try (FileChannel channel = FileChannel.open(Path.of(URI.create(file.getPath())), StandardOpenOption.APPEND)) { + FileLock lock = channel.lock(); if (file.exists()) { FileUtils.delete(file); log.info("System file {} was deleted", file.getName()); } + lock.release(); } catch (IOException e) { log.error("Failed to delete file {}", file.getName(), e); } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java deleted file mode 100644 index 2f2d91a950..0000000000 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.cache.ota.service; - -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.id.OtaPackageId; - -import java.io.*; -import java.sql.Blob; -import java.sql.SQLException; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.CountDownLatch; - -@Slf4j -@RequiredArgsConstructor -@Component -public class BaseFileCacheService implements FileCacheService { - @Value("${java.io.tmpdir}/ota/") - private String PATH; - - private final TemporaryFileCleaner fileCleaner; - private final ConcurrentMap files = new ConcurrentHashMap<>(); - - - public File saveDataTemporaryFile(InputStream inputStream) { - File path = new File(PATH); - try { - File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); - FileUtils.copyInputStreamToFile(inputStream, tempFile); - return tempFile; - } catch (IOException e) { - log.error("Failed to create temp file", e); - throw new RuntimeException("Failed to create temp file for input stream"); - } - } - - public File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException { - String fileName = PATH + otaPackageId.getId(); - if (exist(fileName)) { - fileCleaner.updateFileUsageStatus(otaPackageId); - return new File(fileName); - } - throw new FileNotFoundException(); - } - - public File loadToFile(OtaPackageId otaPackageId, Blob data) { - if (otaPackageId == null || data == null) { - log.error("Received null variables: {}", otaPackageId == null ? "otaPackageId" : "data"); - throw new NullPointerException(); - } - CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data)); - String fileName = PATH + otaPackageId.getId(); - if (latch.getCount() == 0) { - return new File(fileName); - } else { - try { - latch.await(); - return new File(fileName); - } catch (InterruptedException e) { - log.error("Failed to save file {} to file system", fileName, e); - throw new RuntimeException(e); - } - } - } - - public void deleteFile(File file) { - try { - if (file.exists()) { - FileUtils.delete(file); - log.info("System file {} was deleted", file.getName()); - } - } catch (IOException e) { - log.error("Failed to delete file {}", file.getName(), e); - } - } - - public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException { - return new FileInputStream(getOtaDataFile(otaPackageId)); - } - - public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) { - try { - return new FileInputStream(loadToFile(otaPackageId, data)); - } catch (FileNotFoundException e) { - log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e); - throw new RuntimeException(e); - } - } - - private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) { - CountDownLatch latch = new CountDownLatch(1); - try { - InputStream binaryStream = data.getBinaryStream(); - String fileName = PATH + otaPackageId.getId(); - saveAsSystemFile(fileName, binaryStream); - latch.countDown(); - return latch; - } catch (SQLException e) { - log.error("Failed to get binary data from otaPackage {}", otaPackageId, e); - throw new RuntimeException("Failed to get binary data from database"); - } - } - - private void saveAsSystemFile(String fileName, InputStream inputStream) { - try { - File file = new File(fileName); - FileUtils.copyInputStreamToFile(inputStream, file); - } catch (IOException e) { - log.error("Failed to copy stream to system file {}", fileName, e); - throw new RuntimeException("Failed to save file"); - } - } - - private boolean exist(String name) { - File file = new File(name); - return file.exists(); - } -} \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java deleted file mode 100644 index b022932c4f..0000000000 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.thingsboard.server.cache.ota.service; - -import org.thingsboard.server.common.data.id.OtaPackageId; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.InputStream; -import java.sql.Blob; - -public interface FileCacheService { - void deleteFile(File file); - InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException ; - InputStream loadOtaData(OtaPackageId otaPackageId, Blob data); - File loadToFile(OtaPackageId otaPackageId, Blob data); - File saveDataTemporaryFile(InputStream inputStream); - File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException; -} diff --git a/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java index 20cd1dc47b..a0d96b6a3c 100644 --- a/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java +++ b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java @@ -1,19 +1,21 @@ package org.thingsboard.server.cache.ota.service; import lombok.SneakyThrows; -import org.hibernate.engine.jdbc.BlobProxy; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import org.thingsboard.server.cache.ota.files.BaseFileCacheService; +import org.thingsboard.server.cache.ota.files.TemporaryFileCleaner; import org.thingsboard.server.common.data.id.OtaPackageId; -import java.io.*; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; import java.security.MessageDigest; -import java.sql.Blob; import java.util.Objects; import java.util.UUID; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; class BaseFileCacheServiceTest { @@ -21,30 +23,18 @@ class BaseFileCacheServiceTest { private final static String FILE_FILLING = "Hello, testing environment"; private static final int ONE_MEGA_BYTE = 1_000_000; private final static OtaPackageId OTA_PACKAGE_ID = new OtaPackageId(UUID.randomUUID()); - private final static Blob DATA = BlobProxy.generateProxy(FILE_FILLING.getBytes()); - File file; + private final static InputStream DATA = new ByteArrayInputStream(FILE_FILLING.getBytes()); BaseFileCacheService baseFileCacheService = new BaseFileCacheService(new TemporaryFileCleaner()); - @BeforeEach - void setUp() { - file = Mockito.mock(File.class); -// fileCacheService = Mockito.mock(FileCacheService.class); - } - - @Test - void testNotThrowExceptionsWhenDeleteFile() { - Mockito.when(file.exists()).thenReturn(true); - assertDoesNotThrow(() -> baseFileCacheService.deleteFile(new File("/download/alarm_rule.js"))); - } @Test - void testDataSavingWithNullBlob() { - assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, null)); + void testDataSavingWithNullInputStream() { + assertThrows(RuntimeException.class, () -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, null)); } @Test void testDataSavingWithNullOtaPackageId() { - assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(null, DATA)); + assertThrows(RuntimeException.class, () -> baseFileCacheService.loadToFile(null, DATA)); } @Test @@ -71,19 +61,6 @@ class BaseFileCacheServiceTest { assertEquals(sha256, calculateChecksumSHA256(new FileInputStream(file))); } - @Test - @SneakyThrows - void testNotThrowExceptionWhenCreatingTemporaryFiles() { - BaseFileCacheService mock = Mockito.mock(BaseFileCacheService.class); - assertDoesNotThrow(() -> mock.saveDataTemporaryFile(DATA.getBinaryStream())); - } - - @Test - @SneakyThrows - void testGettingStreamWhenFileNotExist(){ - assertThrows(FileNotFoundException.class, ()->baseFileCacheService.getOtaDataStream(OTA_PACKAGE_ID)); - } - @SneakyThrows String calculateChecksumSHA256(InputStream stream) { MessageDigest md = MessageDigest.getInstance("SHA-256"); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index c0c5b64a1f..22769eea6c 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -21,14 +21,12 @@ import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import java.io.File; import java.io.InputStream; -import java.nio.ByteBuffer; public interface OtaPackageService { @@ -36,8 +34,6 @@ public interface OtaPackageService { OtaPackage saveOtaPackage(OtaPackage otaPackage); - String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data); - OtaPackage findOtaPackageById(TenantId tenantId, OtaPackageId otaPackageId); OtaPackageInfo findOtaPackageInfoById(TenantId tenantId, OtaPackageId otaPackageId); @@ -54,6 +50,5 @@ public interface OtaPackageService { long sumDataSizeByTenantId(TenantId tenantId); - InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId); - File getOtaDataFile (TenantId tenantId, OtaPackageId otaPackageId); + File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java index b4957443e9..5c7facbdc5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,24 +15,28 @@ */ package org.thingsboard.server.common.data; +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; import lombok.EqualsAndHashCode; import org.thingsboard.server.common.data.id.OtaPackageId; -import java.nio.ByteBuffer; -import java.sql.Blob; +import java.io.InputStream; @ApiModel @Data @EqualsAndHashCode(callSuper = true) +@JsonIgnoreProperties(value = { "data" }) public class OtaPackage extends OtaPackageInfo { private static final long serialVersionUID = 3091601761339422546L; @ApiModelProperty(position = 16, value = "OTA Package data.", readOnly = true) - private transient Blob data; + private transient InputStream data; public OtaPackage() { super(); diff --git a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java index f9df92966b..b4b09402fe 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/JacksonUtil.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java index f83b768e41..12876f8b55 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,44 +18,32 @@ package org.thingsboard.server.dao.model.sql; import com.fasterxml.jackson.databind.JsonNode; import lombok.Data; import lombok.EqualsAndHashCode; +import lombok.SneakyThrows; +import org.apache.commons.io.FileUtils; +import org.aspectj.util.FileUtil; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; +import org.hibernate.engine.jdbc.BlobProxy; +import org.postgresql.largeobject.BlobInputStream; +import org.springframework.beans.factory.annotation.Value; import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; +import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.model.BaseSqlEntity; import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.model.SearchTextEntity; import org.thingsboard.server.dao.util.mapping.JsonStringType; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.EnumType; -import javax.persistence.Enumerated; -import javax.persistence.Lob; -import javax.persistence.Table; -import java.nio.ByteBuffer; +import javax.persistence.*; +import java.io.File; +import java.io.FileInputStream; import java.sql.Blob; import java.util.UUID; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_CHECKSUM_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_CONTENT_TYPE_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_DATA_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_DATA_SIZE_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_DEVICE_PROFILE_ID_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_FILE_NAME_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TABLE_NAME; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TAG_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TENANT_ID_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TILE_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TYPE_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_URL_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_VERSION_COLUMN; -import static org.thingsboard.server.dao.model.ModelConstants.SEARCH_TEXT_PROPERTY; +import static org.thingsboard.server.dao.model.ModelConstants.*; @Data @EqualsAndHashCode(callSuper = true) @@ -133,7 +121,7 @@ public class OtaPackageEntity extends BaseSqlEntity implements Searc this.contentType = otaPackage.getContentType(); this.checksumAlgorithm = otaPackage.getChecksumAlgorithm(); this.checksum = otaPackage.getChecksum(); - this.data = otaPackage.getData(); + this.data = BlobProxy.generateProxy(otaPackage.getData(), otaPackage.getDataSize()); this.dataSize = otaPackage.getDataSize(); this.additionalInfo = otaPackage.getAdditionalInfo(); } @@ -149,6 +137,7 @@ public class OtaPackageEntity extends BaseSqlEntity implements Searc } @Override + @SneakyThrows public OtaPackage toData() { OtaPackage otaPackage = new OtaPackage(new OtaPackageId(id)); otaPackage.setCreatedTime(createdTime); @@ -167,7 +156,7 @@ public class OtaPackageEntity extends BaseSqlEntity implements Searc otaPackage.setChecksum(checksum); otaPackage.setDataSize(dataSize); if (data != null) { - otaPackage.setData(data); + otaPackage.setData(data.getBinaryStream()); otaPackage.setHasData(true); } otaPackage.setAdditionalInfo(additionalInfo); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 435479932a..f27674f6f1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -15,26 +15,22 @@ */ package org.thingsboard.server.dao.ota; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.hibernate.engine.jdbc.BlobProxy; import org.hibernate.exception.ConstraintViolationException; import org.springframework.cache.Cache; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.ota.OtaPackageDataCache; -import org.thingsboard.server.cache.ota.service.BaseFileCacheService; +import org.thingsboard.server.cache.ota.files.BaseFileCacheService; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -43,9 +39,10 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import javax.transaction.Transactional; -import java.io.*; -import java.nio.ByteBuffer; -import java.sql.SQLException; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -71,7 +68,7 @@ public class BaseOtaPackageService implements OtaPackageService { @Override public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl) { log.trace("Executing saveOtaPackageInfo [{}]", otaPackageInfo); - if(isUrl && (StringUtils.isEmpty(otaPackageInfo.getUrl()) || otaPackageInfo.getUrl().trim().length() == 0)) { + if (isUrl && (StringUtils.isEmpty(otaPackageInfo.getUrl()) || otaPackageInfo.getUrl().trim().length() == 0)) { throw new DataValidationException("Ota package URL should be specified!"); } otaPackageInfoValidator.validate(otaPackageInfo, OtaPackageInfo::getTenantId); @@ -94,29 +91,26 @@ public class BaseOtaPackageService implements OtaPackageService { } @Override + @Transactional public OtaPackage saveOtaPackage(OtaPackage otaPackage) { log.trace("Executing saveOtaPackage [{}]", otaPackage); try { - File tempFile= baseFileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream()); - otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); - try { - otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); - }catch (RuntimeException e){ - baseFileCacheService.deleteFile(tempFile); - throw e; - } - otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); + File tempFile = baseFileCacheService.saveDataTemporaryFile(otaPackage.getData()); + otaPackage.setData(new FileInputStream(tempFile)); + otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); + BufferedInputStream stream = new BufferedInputStream(new FileInputStream(tempFile)); + stream.mark(0); + otaPackage.setData(stream); OtaPackageId otaPackageId = otaPackage.getId(); if (otaPackageId != null) { Cache cache = cacheManager.getCache(OTA_PACKAGE_CACHE); cache.evict(toOtaPackageInfoKey(otaPackageId)); otaPackageDataCache.evict(otaPackageId.toString()); } - OtaPackage save = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); - baseFileCacheService.deleteFile(tempFile); - return save; - } catch (FileNotFoundException | SQLException e){ - log.error("Failed to validate ota package {}",otaPackage.getId(), e); + OtaPackage savedOtaPackage = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); + return savedOtaPackage; + } catch (FileNotFoundException e) { + log.error("Failed to save data of ota package {} to file", otaPackage.getId(), e); throw new RuntimeException(e); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); @@ -127,36 +121,6 @@ public class BaseOtaPackageService implements OtaPackageService { } } } - @Override - public String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data) { - if (data == null || !data.hasArray() || data.array().length == 0) { - throw new DataValidationException("OtaPackage data should be specified!"); - } - - return getHashFunction(checksumAlgorithm).hashBytes(data.array()).toString(); - } - - @SuppressWarnings("deprecation") - private HashFunction getHashFunction(ChecksumAlgorithm checksumAlgorithm) { - switch (checksumAlgorithm) { - case MD5: - return Hashing.md5(); - case SHA256: - return Hashing.sha256(); - case SHA384: - return Hashing.sha384(); - case SHA512: - return Hashing.sha512(); - case CRC32: - return Hashing.crc32(); - case MURMUR3_32: - return Hashing.murmur3_32(); - case MURMUR3_128: - return Hashing.murmur3_128(); - default: - throw new DataValidationException("Unknown checksum algorithm!"); - } - } @Override public OtaPackage findOtaPackageById(TenantId tenantId, OtaPackageId otaPackageId) { @@ -261,31 +225,19 @@ public class BaseOtaPackageService implements OtaPackageService { return Collections.singletonList(otaPackageId); } + @Override @Transactional - public InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId){ - try { - return baseFileCacheService.getOtaDataStream(otaPackageId); - }catch (FileNotFoundException e){ - OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); - if (otaPackage == null) { - log.error("Can't find otaPackage to download file {}", otaPackage); - throw new RuntimeException("No such OtaPackageId"); - } - return baseFileCacheService.loadOtaData(otaPackageId,otaPackage.getData()); + public File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId) { + validateId(otaPackageId, INCORRECT_OTA_PACKAGE_ID + otaPackageId); + Optional otaDataFile = baseFileCacheService.getOtaDataFile(otaPackageId); + if (otaDataFile.isPresent()) { + return otaDataFile.get(); } - } - - @Transactional - public File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId){ - try { - return baseFileCacheService.getOtaDataFile(otaPackageId); - }catch (FileNotFoundException e){ - OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); - if (otaPackage == null) { - log.error("Can't find otaPackage to download file {}", otaPackage); - throw new RuntimeException("No such OtaPackageId"); - } - return baseFileCacheService.loadToFile(otaPackageId,otaPackage.getData()); + OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); + if (otaPackage == null) { + log.error("Can't find otaPackage to download file {}", otaPackageId); + throw new RuntimeException("No such OtaPackageId"); } + return baseFileCacheService.loadToFile(otaPackageId, otaPackage.getData()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java b/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java index 9d3c7469e2..a206e19dd0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java @@ -65,15 +65,13 @@ public class ChecksumUtil { private static String checksumMurmur3_32(InputStream stream) throws IOException { Hasher hasher = Hashing.murmur3_32().newHasher(); - ByteStreams.copy(stream, Funnels.asOutputStream(hasher)); - String hash = new String(hasher.hash().asBytes()); - return new String(Base64.getEncoder().encode(hash.getBytes())); + com.google.common.io.ByteStreams.copy(stream, Funnels.asOutputStream(hasher)); + return hasher.hash().toString(); } private static String checksumMurmur3_128(InputStream stream) throws IOException { Hasher hasher = Hashing.murmur3_128().newHasher(); ByteStreams.copy(stream, Funnels.asOutputStream(hasher)); - String hash = new String(hasher.hash().asBytes()); - return new String(Base64.getEncoder().encode(hash.getBytes())); + return hasher.hash().toString(); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java index e55b5e333a..a24920f106 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,14 +25,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.ota.OtaPackageDao; -import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; - -import java.sql.SQLException; - - import static org.thingsboard.server.common.data.EntityType.OTA_PACKAGE; @Component @@ -73,14 +68,9 @@ public class OtaPackageDataValidator extends BaseOtaPackageDataValidatorChecksumUtil.generateChecksum(MURMUR3_32, inputStream)); + void testMurmur3_32Checksum() { + assertEquals("3d45a1dd", ChecksumUtil.generateChecksum(MURMUR3_32, inputStream)); + assertDoesNotThrow(() -> ChecksumUtil.generateChecksum(MURMUR3_32, inputStream)); } @Test - void testMurmur3_128Checksum(){ - assertDoesNotThrow(()->ChecksumUtil.generateChecksum(MURMUR3_128, inputStream)); + void testMurmur3_128Checksum() { + assertEquals("9dbf0ffe5f6ecbb5edf207717e7870b7", ChecksumUtil.generateChecksum(MURMUR3_128, inputStream)); + assertDoesNotThrow(() -> ChecksumUtil.generateChecksum(MURMUR3_128, inputStream)); } } \ No newline at end of file diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java index 03783cf833..e3270d2d67 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java @@ -74,6 +74,7 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.dao.widget.WidgetTypeService; import org.thingsboard.server.dao.widget.WidgetsBundleService; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Comparator; @@ -269,7 +270,7 @@ public abstract class AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(BlobProxy.generateProxy(new byte[]{1})); + firmware.setData(new ByteArrayInputStream(new byte[]{1})); firmware.setDataSize(1L); return firmware; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java index 400286c13e..e7528f13ee 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java @@ -38,6 +38,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.exception.DataValidationException; +import java.io.ByteArrayInputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -111,7 +112,7 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(BlobProxy.generateProxy(new byte[]{1})); + firmware.setData(new ByteArrayInputStream(new byte[]{1})); firmware.setDataSize(1L); OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java index 283ac8fd0e..acd7e0f229 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java @@ -42,6 +42,7 @@ import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; +import java.io.ByteArrayInputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; @@ -201,7 +202,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(BlobProxy.generateProxy(new byte[]{1})); + firmware.setData(new ByteArrayInputStream(new byte[]{1})); firmware.setDataSize(1L); OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); @@ -236,7 +237,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { firmware.setContentType("text/plain"); firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); - firmware.setData(BlobProxy.generateProxy(new byte[]{1})); + firmware.setData(new ByteArrayInputStream(new byte[]{1})); firmware.setDataSize(1L); OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 1e0b0340c0..ac27c93054 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -32,10 +32,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.ota.util.ChecksumUtil; import javax.validation.ValidationException; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.InputStream; +import java.io.*; import java.sql.Blob; import java.util.ArrayList; import java.util.Collections; @@ -54,7 +51,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { private static final ChecksumAlgorithm CHECKSUM_ALGORITHM = ChecksumAlgorithm.SHA256; private static final String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"; private static final long DATA_SIZE = 1L; - private static final Blob DATA = BlobProxy.generateProxy(new byte[]{(int) DATA_SIZE}); + private static final InputStream DATA = new ByteArrayInputStream(new byte[]{1}); private static final String URL = "http://firmware.test.org"; private final IdComparator idComparator = new IdComparator<>(); @@ -693,13 +690,6 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { otaPackageService.saveOtaPackageInfo(firmwareInfo, true); } - @Test - public void testGettingCorrectStreamWithOtaData() { - OtaPackage firmware = createFirmware(tenantId, "24687846"); - InputStream stream = otaPackageService.getOtaDataStream(tenantId, firmware.getId()); - assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, stream)); - } - @Test public void testGettingCorrectFileWithOtaData() { OtaPackage firmware = createFirmware(tenantId, "24687846"); From e725fb8200f4101bc3102498674c98f5e4672bc3 Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Mon, 8 Aug 2022 16:31:20 +0300 Subject: [PATCH 06/11] headers --- .../server/controller/OtaPackageController.java | 8 ++++---- .../service/action/EntityActionService.java | 8 ++++---- .../server/cache/ota/CaffeineOtaPackageCache.java | 8 ++++---- .../cache/ota/files/BaseFileCacheService.java | 8 ++++---- .../server/cache/ota/files/FileCacheService.java | 15 +++++++++++++++ .../cache/ota/files/TemporaryFileCleaner.java | 15 +++++++++++++++ .../ota/service/BaseFileCacheServiceTest.java | 15 +++++++++++++++ .../server/dao/ota/OtaPackageService.java | 8 ++++---- .../server/common/data/OtaPackage.java | 8 ++++---- .../server/common/data/ota/ChecksumAlgorithm.java | 8 ++++---- .../server/dao/model/sql/OtaPackageEntity.java | 8 ++++---- .../server/dao/ota/BaseOtaPackageService.java | 8 ++++---- .../server/dao/ota/util/ChecksumUtil.java | 15 +++++++++++++++ .../validator/OtaPackageDataValidator.java | 8 ++++---- .../server/dao/ota/util/ChecksumUtilTest.java | 15 +++++++++++++++ .../dao/service/BaseOtaPackageServiceTest.java | 8 ++++---- 16 files changed, 119 insertions(+), 44 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index da758f5f27..70a3d38148 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java index bc17d0fc93..212e957c6b 100644 --- a/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java +++ b/application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java index 5c4e5c2e66..7ea7b18f43 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/CaffeineOtaPackageCache.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java index bbd0056e92..2132bfa0ac 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java index 4575a3502c..cda01b2014 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.cache.ota.files; import org.thingsboard.server.common.data.id.OtaPackageId; diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java index f9a8363376..11ee62c69c 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.cache.ota.files; import lombok.extern.slf4j.Slf4j; diff --git a/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java index a0d96b6a3c..455269773c 100644 --- a/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java +++ b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.cache.ota.service; import lombok.SneakyThrows; diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index 22769eea6c..f8c9c452b0 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java index 5c7facbdc5..b5e9f7d1a9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java b/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java index 73477b5cc6..206e7840cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java index 12876f8b55..be4d4560a6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index f27674f6f1..342e668b85 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java b/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java index a206e19dd0..7000a9ed9e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.ota.util; import com.datastax.oss.driver.shaded.guava.common.io.ByteStreams; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java index a24920f106..96295253f8 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/dao/src/test/java/org/thingsboard/server/dao/ota/util/ChecksumUtilTest.java b/dao/src/test/java/org/thingsboard/server/dao/ota/util/ChecksumUtilTest.java index ba9fac61c3..29294279bb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/ota/util/ChecksumUtilTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/ota/util/ChecksumUtilTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.server.dao.ota.util; import org.junit.jupiter.api.BeforeEach; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index ac27c93054..1734b1563d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - *

+ * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

+ * + * http://www.apache.org/licenses/LICENSE-2.0 + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. From 895434514f6dbf9f12d424f921ee196d17bdc82f Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Tue, 9 Aug 2022 11:02:38 +0300 Subject: [PATCH 07/11] scheduler fixed delay improved --- application/src/main/resources/thingsboard.yml | 2 +- .../server/cache/ota/files/TemporaryFileCleaner.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 79b33785c1..99cf5e9af1 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1116,4 +1116,4 @@ management: include: '${METRICS_ENDPOINTS_EXPOSE:info}' files: - temporary_files_directory: '${TEMPORARY_FILES_DIRECTORY: ${java.io.tmpdir}}' \ No newline at end of file + temporary_files_directory: "${TEMPORARY_FILES_DIRECTORY: ${java.io.tmpdir}}" \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java index 11ee62c69c..77298dd3b9 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java @@ -80,7 +80,7 @@ public class TemporaryFileCleaner { } } - @Scheduled(fixedDelay = 60000) + @Scheduled(fixedDelay = 600_000) private void deleteUnusedTemporaryFiles() { long currentTime = System.currentTimeMillis(); List toBeDeleted = lastActivityTimes.entrySet() From e3a61d3d99948523a60461f8b74548bcdbe61050 Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Tue, 9 Aug 2022 17:08:19 +0300 Subject: [PATCH 08/11] merge with updated master --- .../org/thingsboard/server/controller/OtaPackageController.java | 2 -- .../org/thingsboard/server/dao/ota/BaseOtaPackageService.java | 2 -- 2 files changed, 4 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index 3e6faeebb4..87139aef10 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -190,10 +190,8 @@ public class OtaPackageController extends BaseController { checksum = ChecksumUtil.generateChecksum(checksumAlgorithm, file.getInputStream()); } OtaPackageInfo savedOtaPackage = saveOtaPackageWithData(otaPackageId, file, checksum, checksumAlgorithm); - logEntityAction(savedOtaPackage.getId(), savedOtaPackage, null, ActionType.UPDATED, null); return savedOtaPackage; } catch (Exception e) { - logEntityAction(emptyId(EntityType.OTA_PACKAGE), null, null, ActionType.UPDATED, e, strOtaPackageId); throw handleException(e); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index c45192e2ad..c655bd8df9 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -29,7 +29,6 @@ import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -47,7 +46,6 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; From dc89c871598ccc332d870684ecb00f4b42026f4b Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Wed, 10 Aug 2022 10:43:41 +0300 Subject: [PATCH 09/11] Changed TbOtaPackageService --- .../controller/OtaPackageController.java | 36 +++---------------- .../constructor/OtaPackageMsgConstructor.java | 8 ++++- .../ota/DefaultTbOtaPackageService.java | 35 +++++++++--------- .../entitiy/ota/TbOtaPackageService.java | 3 +- 4 files changed, 31 insertions(+), 51 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index 87139aef10..5e1d377b83 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -179,41 +179,13 @@ public class OtaPackageController extends BaseController { @ApiParam(value = "OTA Package checksum algorithm.", allowableValues = OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES) @RequestParam(CHECKSUM_ALGORITHM) String checksumAlgorithmStr, @ApiParam(value = "OTA Package data.") - @RequestPart MultipartFile file) throws ThingsboardException, IOException { + @RequestPart MultipartFile file) throws ThingsboardException { checkParameter(OTA_PACKAGE_ID, strOtaPackageId); checkParameter(CHECKSUM_ALGORITHM, checksumAlgorithmStr); - try { - OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId)); - ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.valueOf(checksumAlgorithmStr.toUpperCase()); - - if (StringUtils.isEmpty(checksum)) { - checksum = ChecksumUtil.generateChecksum(checksumAlgorithm, file.getInputStream()); - } - OtaPackageInfo savedOtaPackage = saveOtaPackageWithData(otaPackageId, file, checksum, checksumAlgorithm); - return savedOtaPackage; - } catch (Exception e) { - throw handleException(e); - } - } - - private OtaPackage saveOtaPackageWithData(OtaPackageId otaPackageId, MultipartFile file, String checksum, ChecksumAlgorithm checksumAlgorithm) throws ThingsboardException, IOException { + OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId)); OtaPackageInfo info = checkOtaPackageInfoId(otaPackageId, Operation.READ); - OtaPackage otaPackage = new OtaPackage(otaPackageId); - otaPackage.setCreatedTime(info.getCreatedTime()); - otaPackage.setTenantId(info.getTenantId()); - otaPackage.setDeviceProfileId(info.getDeviceProfileId()); - otaPackage.setType(info.getType()); - otaPackage.setTitle(info.getTitle()); - otaPackage.setVersion(info.getVersion()); - otaPackage.setTag(info.getTag()); - otaPackage.setAdditionalInfo(info.getAdditionalInfo()); - otaPackage.setChecksumAlgorithm(checksumAlgorithm); - otaPackage.setChecksum(checksum); - otaPackage.setFileName(file.getOriginalFilename()); - otaPackage.setContentType(file.getContentType()); - otaPackage.setDataSize(file.getSize()); - otaPackage.setData(file.getInputStream()); - return otaPackageService.saveOtaPackage(otaPackage); + ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.valueOf(checksumAlgorithmStr.toUpperCase()); + return tbOtaPackageService.saveOtaPackageData(info, checksum, checksumAlgorithm, file, getCurrentUser()); } @ApiOperation(value = "Get OTA Package Infos (getOtaPackages)", diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java index be38079d1e..cfbd129aa6 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java @@ -24,6 +24,8 @@ import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; +import java.io.IOException; + @Component @TbCoreComponent public class OtaPackageMsgConstructor { @@ -62,7 +64,11 @@ public class OtaPackageMsgConstructor { builder.setDataSize(otaPackage.getDataSize()); } if (otaPackage.getData() != null) { - builder.setData(ByteString.copyFrom(otaPackage.getData().array())); + try { + builder.setData(ByteString.copyFrom(otaPackage.getData().readAllBytes())); + } catch (IOException e){ + throw new RuntimeException(e); + } } return builder.build(); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java index 00cf70db0e..f74666f7ab 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,22 +18,19 @@ package org.thingsboard.server.service.entitiy.ota; import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.OtaPackageInfo; -import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; -import org.thingsboard.server.common.data.StringUtils; -import org.thingsboard.server.common.data.User; +import org.springframework.web.multipart.MultipartFile; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; -import java.nio.ByteBuffer; +import java.io.IOException; @Service @TbCoreComponent @@ -64,12 +61,12 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen @Override public OtaPackageInfo saveOtaPackageData(OtaPackageInfo otaPackageInfo, String checksum, ChecksumAlgorithm checksumAlgorithm, - byte[] data, String filename, String contentType, User user) throws ThingsboardException { + MultipartFile file, User user) throws ThingsboardException { TenantId tenantId = otaPackageInfo.getTenantId(); OtaPackageId otaPackageId = otaPackageInfo.getId(); try { if (StringUtils.isEmpty(checksum)) { - checksum = otaPackageService.generateChecksum(checksumAlgorithm, ByteBuffer.wrap(data)); + checksum = ChecksumUtil.generateChecksum(checksumAlgorithm, file.getInputStream()); } OtaPackage otaPackage = new OtaPackage(otaPackageId); otaPackage.setCreatedTime(otaPackageInfo.getCreatedTime()); @@ -82,14 +79,18 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen otaPackage.setAdditionalInfo(otaPackageInfo.getAdditionalInfo()); otaPackage.setChecksumAlgorithm(checksumAlgorithm); otaPackage.setChecksum(checksum); - otaPackage.setFileName(filename); - otaPackage.setContentType(contentType); - otaPackage.setData(ByteBuffer.wrap(data)); - otaPackage.setDataSize((long) data.length); + otaPackage.setFileName(file.getOriginalFilename()); + otaPackage.setContentType(file.getContentType()); + otaPackage.setData(file.getInputStream()); + otaPackage.setDataSize(file.getSize()); OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, savedOtaPackage.getId(), savedOtaPackage, user, ActionType.UPDATED, true, null); return savedOtaPackage; + } catch (IOException e){ + notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.OTA_PACKAGE), ActionType.UPDATED, + user, e, otaPackageId.toString()); + throw new RuntimeException(e); } catch (Exception e) { notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.OTA_PACKAGE), ActionType.UPDATED, user, e, otaPackageId.toString()); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/TbOtaPackageService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/TbOtaPackageService.java index aec2f20d0c..635a1a4ed6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/TbOtaPackageService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/TbOtaPackageService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.entitiy.ota; +import org.springframework.web.multipart.MultipartFile; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; import org.thingsboard.server.common.data.User; @@ -26,7 +27,7 @@ public interface TbOtaPackageService { OtaPackageInfo save(SaveOtaPackageInfoRequest saveOtaPackageInfoRequest, User user) throws ThingsboardException; OtaPackageInfo saveOtaPackageData(OtaPackageInfo otaPackageInfo, String checksum, ChecksumAlgorithm checksumAlgorithm, - byte[] data, String filename, String contentType, User user) throws ThingsboardException; + MultipartFile file, User user) throws ThingsboardException; void delete(OtaPackageInfo otaPackageInfo, User user) throws ThingsboardException; From 17705dec080c1c378bf4cbd714cdc91f5b2ba736 Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Fri, 12 Aug 2022 16:36:10 +0300 Subject: [PATCH 10/11] improved downloading ota in deviceTransportAPi --- .../ota/DefaultTbOtaPackageService.java | 12 ++- .../service/ota/TbMultipartFileImp.java | 42 +++++++++ .../transport/DefaultTransportApiService.java | 20 +++-- .../sync/ie/BaseExportImportServiceTest.java | 44 ++++++++- .../cache/ota/files/TemporaryFileCleaner.java | 16 ++-- .../server/dao/ota/OtaPackageService.java | 3 +- .../server/dao/ota/TbMultipartFile.java | 15 ++++ .../transport/http/DeviceApiController.java | 29 ++++-- .../server/dao/ota/BaseOtaPackageService.java | 23 ++--- .../AbstractHasOtaPackageValidator.java | 19 ++-- .../service/BaseDeviceProfileServiceTest.java | 43 ++++++++- .../dao/service/BaseDeviceServiceTest.java | 57 ++++++++++-- .../service/BaseOtaPackageServiceTest.java | 90 ++++++++++++++----- .../dao/service/BaseTenantServiceTest.java | 45 ++++++++-- .../service/sql/OtaPackageServiceSqlTest.java | 1 + 15 files changed, 374 insertions(+), 85 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/ota/TbMultipartFileImp.java create mode 100644 common/dao-api/src/main/java/org/thingsboard/server/dao/ota/TbMultipartFile.java diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java index f74666f7ab..ad58df004a 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java @@ -19,7 +19,12 @@ import lombok.AllArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import org.thingsboard.server.common.data.*; +import org.thingsboard.server.common.data.OtaPackageInfo; +import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.OtaPackageId; @@ -29,6 +34,7 @@ import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.AbstractTbEntityService; +import org.thingsboard.server.service.ota.TbMultipartFileImp; import java.io.IOException; @@ -61,7 +67,7 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen @Override public OtaPackageInfo saveOtaPackageData(OtaPackageInfo otaPackageInfo, String checksum, ChecksumAlgorithm checksumAlgorithm, - MultipartFile file, User user) throws ThingsboardException { + MultipartFile file, User user) { TenantId tenantId = otaPackageInfo.getTenantId(); OtaPackageId otaPackageId = otaPackageInfo.getId(); try { @@ -83,7 +89,7 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen otaPackage.setContentType(file.getContentType()); otaPackage.setData(file.getInputStream()); otaPackage.setDataSize(file.getSize()); - OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage); + OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage, new TbMultipartFileImp(file)); notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, savedOtaPackage.getId(), savedOtaPackage, user, ActionType.UPDATED, true, null); return savedOtaPackage; diff --git a/application/src/main/java/org/thingsboard/server/service/ota/TbMultipartFileImp.java b/application/src/main/java/org/thingsboard/server/service/ota/TbMultipartFileImp.java new file mode 100644 index 0000000000..79cc158bea --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/ota/TbMultipartFileImp.java @@ -0,0 +1,42 @@ +package org.thingsboard.server.service.ota; + +import lombok.RequiredArgsConstructor; +import org.springframework.web.multipart.MultipartFile; +import org.thingsboard.server.dao.ota.TbMultipartFile; + +import javax.validation.constraints.NotNull; +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +@RequiredArgsConstructor +public class TbMultipartFileImp implements TbMultipartFile { + + @NotNull + private final MultipartFile file; + + @Override + public Optional getInputStream() { + try { + return Optional.of(file.getInputStream()); + } catch (IOException e) { + return Optional.empty(); + } + + } + + @Override + public String getFileName() { + return file.getName(); + } + + @Override + public long getFileSize() { + return file.getSize(); + } + + @Override + public String getContentType() { + return file.getContentType(); + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index edf43a5c74..e8da713c7d 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -101,6 +101,7 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.resource.TbResourceService; +import java.io.File; import java.io.IOException; import java.util.List; import java.util.Optional; @@ -599,16 +600,17 @@ public class DefaultTransportApiService implements TransportApiService { builder.setType(otaPackageInfo.getType().name()); builder.setTitle(otaPackageInfo.getTitle()); builder.setVersion(otaPackageInfo.getVersion()); - builder.setFileName(otaPackageInfo.getFileName()); + File file = otaPackageService.getOtaDataFile(otaPackageInfo.getTenantId(), otaPackageId); + builder.setFileName(file.getAbsolutePath()); builder.setContentType(otaPackageInfo.getContentType()); - if (!otaPackageDataCache.has(otaPackageId.toString())) { - OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); - try { - otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().readAllBytes()); - } catch (IOException e) { - log.error("Failed to cache ota package with id {}",otaPackage.getId(), e); - } - } +// if (!otaPackageDataCache.has(otaPackageId.toString())) { +// OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); +// try { +// otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().readAllBytes()); +// } catch (IOException e) { +// log.error("Failed to cache ota package with id {}",otaPackage.getId(), e); +// } +// } } } diff --git a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java index e516d5e131..fbe5fa2310 100644 --- a/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import org.junit.After; import org.junit.Before; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.mock.web.MockMultipartFile; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode; import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration; @@ -71,17 +72,23 @@ import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.entityview.EntityViewService; import org.thingsboard.server.dao.ota.OtaPackageService; +import org.thingsboard.server.dao.ota.TbMultipartFile; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.rule.RuleChainService; +import org.thingsboard.server.dao.service.BaseOtaPackageServiceTest; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Collections; +import java.util.Optional; import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -171,8 +178,9 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest otaPackage.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); otaPackage.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); otaPackage.setDataSize(1L); - otaPackage.setData(ByteBuffer.wrap(new byte[]{(int) 1})); - return otaPackageService.saveOtaPackage(otaPackage); + otaPackage.setData(new ByteArrayInputStream(new byte[]{1})); + MockMultipartFile file = new MockMultipartFile("filename.txt", new byte[]{1}); + return otaPackageService.saveOtaPackage(otaPackage, new TestTbMultipartFile(file)); } protected void checkImportedDeviceData(Device initialDevice, Device importedDevice) { @@ -449,4 +457,36 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest return new SecurityUser(user, true, new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail())); } + private class TestTbMultipartFile implements TbMultipartFile { + private final MockMultipartFile file; + + private TestTbMultipartFile(MockMultipartFile file) { + this.file = file; + } + + @Override + public Optional getInputStream() { + try { + return Optional.of(file.getInputStream()); + } catch (IOException e) { + return Optional.empty(); + } + } + + @Override + public String getFileName() { + return file.getName(); + } + + @Override + public long getFileSize() { + return file.getSize(); + + } + + @Override + public String getContentType() { + return file.getContentType(); + } + } } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java index 77298dd3b9..92f19189bf 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java @@ -43,6 +43,7 @@ import java.util.stream.Collectors; public class TemporaryFileCleaner { @Value("${files.temporary_files_directory}/ota/") private String PATH; + private final static String FILE_NAME_TEMPLATE = "%s%s.tmp"; private final static long TEMPORARY_FILE_INACTIVITY_TIME = 900_000; private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); @@ -51,19 +52,24 @@ public class TemporaryFileCleaner { } @EventListener(ApplicationReadyEvent.class) - public void createScheduledTaskToClear() { + public void cleanDirectoryWithTemporaryFiles() { createTempDirectoryIfNotExist(); - cleanDirectoryWithTemporaryFiles(); + cleanDirectory(); + log.info("Directory {} with temporary ota files cleaned", PATH); } private void createTempDirectoryIfNotExist() { File directory = new File(PATH); if (!directory.exists()) { - directory.mkdir(); + try{ + FileUtils.forceMkdir(directory); + } catch(IOException e){ + log.error("Failed to create directory for temporary files ", e); + } } } - private void cleanDirectoryWithTemporaryFiles() { + private void cleanDirectory() { File directory = new File(PATH); if (directory.isDirectory()) { File[] files = directory.listFiles(); @@ -100,7 +106,7 @@ public class TemporaryFileCleaner { } private synchronized void deleteFile(String otaId) { - String fileName = PATH + otaId; + String fileName = String.format(FILE_NAME_TEMPLATE, PATH, otaId); File file = new File(fileName); try (FileChannel channel = FileChannel.open(Path.of(URI.create(file.getPath())), StandardOpenOption.APPEND)) { FileLock lock = channel.lock(); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index f8c9c452b0..304adba019 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.ota; import com.google.common.util.concurrent.ListenableFuture; +import org.springframework.boot.web.servlet.MultipartConfigFactory; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -32,7 +33,7 @@ public interface OtaPackageService { OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl); - OtaPackage saveOtaPackage(OtaPackage otaPackage); + OtaPackage saveOtaPackage(OtaPackage otaPackage, TbMultipartFile file); OtaPackage findOtaPackageById(TenantId tenantId, OtaPackageId otaPackageId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/TbMultipartFile.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/TbMultipartFile.java new file mode 100644 index 0000000000..1b92c6405f --- /dev/null +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/TbMultipartFile.java @@ -0,0 +1,15 @@ +package org.thingsboard.server.dao.ota; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; + +public interface TbMultipartFile { + Optional getInputStream(); + + String getFileName(); + + long getFileSize(); + + String getContentType(); +} diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index de9619cb9f..ca991ff7de 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -20,10 +20,13 @@ import com.google.gson.JsonParser; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.core.io.ByteArrayResource; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -65,6 +68,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseM import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import javax.servlet.http.HttpServletRequest; +import java.io.File; +import java.io.FileInputStream; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -505,15 +510,25 @@ public class DeviceApiController implements TbTransportService { } @Override + @SneakyThrows public void onSuccess(TransportProtos.GetOtaPackageResponseMsg otaPackageResponseMsg) { if (!TransportProtos.ResponseStatus.SUCCESS.equals(otaPackageResponseMsg.getResponseStatus())) { responseWriter.setResult(new ResponseEntity<>(HttpStatus.NOT_FOUND)); } else if (title.equals(otaPackageResponseMsg.getTitle()) && version.equals(otaPackageResponseMsg.getVersion())) { - String otaPackageId = new UUID(otaPackageResponseMsg.getOtaPackageIdMSB(), otaPackageResponseMsg.getOtaPackageIdLSB()).toString(); - ByteArrayResource resource = new ByteArrayResource(transportContext.getOtaPackageDataCache().get(otaPackageId, chuckSize, chuck)); - ResponseEntity response = ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackageResponseMsg.getFileName()) - .header("x-filename", otaPackageResponseMsg.getFileName()) +// String otaPackageId = new UUID(otaPackageResponseMsg.getOtaPackageIdMSB(), otaPackageResponseMsg.getOtaPackageIdLSB()).toString(); + File file = new File(otaPackageResponseMsg.getFileName()); + Resource resource = null; + if(chuckSize!=0){ + FileInputStream fileInputStream = new FileInputStream(file); + byte[] bytes = new byte[chuckSize]; + fileInputStream.read(bytes, chuck, chuckSize); + resource = new ByteArrayResource(bytes); + } else { + resource= new FileSystemResource(file); + } + ResponseEntity response = ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + getFileName(otaPackageResponseMsg.getFileName())) + .header("x-filename", getFileName(otaPackageResponseMsg.getFileName())) .contentLength(resource.contentLength()) .contentType(parseMediaType(otaPackageResponseMsg.getContentType())) .body(resource); @@ -523,6 +538,10 @@ public class DeviceApiController implements TbTransportService { } } + private String getFileName(String fileName) { + return new File(fileName).getName(); + } + @Override public void onError(Throwable e) { log.warn("Failed to process request", e); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index c655bd8df9..d7db3d30cd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -38,10 +38,7 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import javax.transaction.Transactional; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; +import java.io.*; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -98,25 +95,23 @@ public class BaseOtaPackageService extends AbstractCachedEntityService optionalStream = file.getInputStream(); + if(optionalStream.isEmpty()){ + log.error("Failed to get input stream from file {}", file.getFileName()); + throw new RuntimeException("Failed to save ota package file"); + } + BufferedInputStream stream = new BufferedInputStream(optionalStream.get()); stream.mark(0); otaPackage.setData(stream); OtaPackageId otaPackageId = otaPackage.getId(); if (otaPackageId != null) { publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); } - OtaPackage savedOtaPackage = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); - return savedOtaPackage; - } catch (FileNotFoundException e) { - log.error("Failed to save data of ota package {} to file", otaPackage.getId(), e); - throw new RuntimeException(e); + return otaPackageDao.save(otaPackage.getTenantId(), otaPackage); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("ota_package_tenant_title_version_unq_key")) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AbstractHasOtaPackageValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AbstractHasOtaPackageValidator.java index 069e1fcfc8..6ba4b40012 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/AbstractHasOtaPackageValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/AbstractHasOtaPackageValidator.java @@ -20,6 +20,7 @@ import org.springframework.context.annotation.Lazy; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.HasOtaPackage; import org.thingsboard.server.common.data.OtaPackage; +import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.OtaPackageType; @@ -35,29 +36,29 @@ public abstract class AbstractHasOtaPackageValidator> exte protected void validateOtaPackage(TenantId tenantId, T entity, DeviceProfileId deviceProfileId) { if (entity.getFirmwareId() != null) { - OtaPackage firmware = otaPackageService.findOtaPackageById(tenantId, entity.getFirmwareId()); + OtaPackageInfo firmware = otaPackageService.findOtaPackageInfoById(tenantId, entity.getFirmwareId()); validateOtaPackage(tenantId, OtaPackageType.FIRMWARE, deviceProfileId, firmware); } if (entity.getSoftwareId() != null) { - OtaPackage software = otaPackageService.findOtaPackageById(tenantId, entity.getSoftwareId()); + OtaPackageInfo software = otaPackageService.findOtaPackageInfoById(tenantId, entity.getSoftwareId()); validateOtaPackage(tenantId, OtaPackageType.SOFTWARE, deviceProfileId, software); } } - private void validateOtaPackage(TenantId tenantId, OtaPackageType type, DeviceProfileId deviceProfileId, OtaPackage otaPackage) { - if (otaPackage == null) { + private void validateOtaPackage(TenantId tenantId, OtaPackageType type, DeviceProfileId deviceProfileId, OtaPackageInfo otaPackageInfo) { + if (otaPackageInfo == null) { throw new DataValidationException(prepareMsg("Can't assign non-existent %s!", type)); } - if (!otaPackage.getTenantId().equals(tenantId)) { + if (!otaPackageInfo.getTenantId().equals(tenantId)) { throw new DataValidationException(prepareMsg("Can't assign %s from different tenant!", type)); } - if (!otaPackage.getType().equals(type)) { - throw new DataValidationException(prepareMsg("Can't assign %s with type: " + otaPackage.getType(), type)); + if (!otaPackageInfo.getType().equals(type)) { + throw new DataValidationException(prepareMsg("Can't assign %s with type: " + otaPackageInfo.getType(), type)); } - if (otaPackage.getData() == null && !otaPackage.hasUrl()) { + if (otaPackageInfo.getDataSize() == 0 && !otaPackageInfo.hasUrl()) { throw new DataValidationException(prepareMsg("Can't assign %s with empty data!", type)); } - if (!otaPackage.getDeviceProfileId().equals(deviceProfileId)) { + if (!otaPackageInfo.getDeviceProfileId().equals(deviceProfileId)) { throw new DataValidationException(prepareMsg("Can't assign %s with different deviceProfile!", type)); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java index 0f84dbb9b7..309bd65d87 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java @@ -25,6 +25,7 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Ignore; import org.junit.Test; +import org.springframework.mock.web.MockMultipartFile; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -37,12 +38,16 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.TbMultipartFile; import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -115,7 +120,8 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest { firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); firmware.setData(new ByteArrayInputStream(new byte[]{1})); firmware.setDataSize(1L); - OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile("test.txt", new byte[]{1}); + OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); deviceProfile.setFirmwareId(savedFirmware.getId()); @@ -261,7 +267,8 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest { DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile"); deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile); OtaPackage otaPackage = constructDefaultOtaPackage(tenantId, deviceProfile.getId()); - otaPackage = otaPackageService.saveOtaPackage(otaPackage); + MockMultipartFile file = new MockMultipartFile("filename.txt", new byte[]{1}); + otaPackage = otaPackageService.saveOtaPackage(otaPackage, new TestTbMultipartFile(file)); assertThat(deviceProfileService.findDeviceProfileById(tenantId, deviceProfile.getId())).isNotNull(); assertThat(otaPackageService.findOtaPackageById(tenantId, otaPackage.getId())).isNotNull(); @@ -372,4 +379,36 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest { Assert.assertEquals(1, pageData.getTotalElements()); } + private class TestTbMultipartFile implements TbMultipartFile { + private final MockMultipartFile file; + + private TestTbMultipartFile(MockMultipartFile file) { + this.file = file; + } + + @Override + public Optional getInputStream() { + try { + return Optional.of(file.getInputStream()); + } catch (IOException e) { + return Optional.empty(); + } + } + + @Override + public String getFileName() { + return file.getName(); + } + + @Override + public long getFileSize() { + return file.getSize(); + + } + + @Override + public String getContentType() { + return file.getContentType(); + } + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java index acd7e0f229..2ca592d936 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java @@ -17,13 +17,13 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; import org.apache.commons.lang3.RandomStringUtils; -import org.hibernate.engine.jdbc.BlobProxy; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.springframework.mock.web.MockMultipartFile; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceInfo; @@ -41,12 +41,15 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.TbMultipartFile; import java.io.ByteArrayInputStream; -import java.nio.ByteBuffer; +import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; @@ -204,7 +207,8 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); firmware.setData(new ByteArrayInputStream(new byte[]{1})); firmware.setDataSize(1L); - OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile("test.txt", new byte[]{1}); + OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); savedDevice.setFirmwareId(savedFirmware.getId()); @@ -239,7 +243,9 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); firmware.setData(new ByteArrayInputStream(new byte[]{1})); firmware.setDataSize(1L); - OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile("test.txt", new byte[]{1}); + OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); + savedDevice.setFirmwareId(savedFirmware.getId()); @@ -848,8 +854,8 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { deviceInfosWithLabel.stream() .anyMatch( d -> d.getId().equals(savedDevice.getId()) - && d.getTenantId().equals(tenantId) - && d.getLabel().equals(savedDevice.getLabel()) + && d.getTenantId().equals(tenantId) + && d.getLabel().equals(savedDevice.getLabel()) ) ); @@ -907,9 +913,9 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { deviceInfosWithLabel.stream() .anyMatch( d -> d.getId().equals(savedDevice.getId()) - && d.getTenantId().equals(tenantId) - && d.getDeviceProfileName().equals(savedDevice.getType()) - && d.getLabel().equals(savedDevice.getLabel()) + && d.getTenantId().equals(tenantId) + && d.getDeviceProfileName().equals(savedDevice.getType()) + && d.getLabel().equals(savedDevice.getLabel()) ) ); @@ -975,4 +981,37 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { ) ); } + + private class TestTbMultipartFile implements TbMultipartFile { + private final MockMultipartFile file; + + private TestTbMultipartFile(MockMultipartFile file) { + this.file = file; + } + + @Override + public Optional getInputStream() { + try { + return Optional.of(file.getInputStream()); + } catch (IOException e) { + return Optional.empty(); + } + } + + @Override + public String getFileName() { + return file.getName(); + } + + @Override + public long getFileSize() { + return file.getSize(); + + } + + @Override + public String getContentType() { + return file.getContentType(); + } + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 00beb835d9..3c26c35eec 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -23,6 +23,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import org.springframework.mock.web.MockMultipartFile; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; @@ -37,6 +38,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.TbMultipartFile; import org.thingsboard.server.dao.ota.util.ChecksumUtil; import javax.validation.ValidationException; @@ -45,8 +47,10 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.InstanceOfAssertFactories.FILE; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; @@ -140,7 +144,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { firmware.setChecksum(CHECKSUM); firmware.setData(DATA); firmware.setDataSize(DATA_SIZE); - OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); Assert.assertNotNull(savedFirmware); Assert.assertNotNull(savedFirmware.getId()); @@ -152,7 +157,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { Assert.assertEquals(firmware.getData(), savedFirmware.getData()); savedFirmware.setAdditionalInfo(JacksonUtil.newObjectNode()); - otaPackageService.saveOtaPackage(savedFirmware); + otaPackageService.saveOtaPackage(savedFirmware, new TestTbMultipartFile(file)); OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId()); Assert.assertEquals(foundFirmware.getTitle(), savedFirmware.getTitle()); @@ -219,7 +224,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { firmware.setData(DATA); firmware.setDataSize(DATA_SIZE); - otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); savedFirmwareInfo = otaPackageService.findOtaPackageInfoById(tenantId, savedFirmwareInfo.getId()); savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode()); @@ -249,7 +255,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage should be assigned to tenant!"); - otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test @@ -267,7 +274,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("Type should be specified!"); - otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test @@ -285,7 +293,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage title should be specified!"); - otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test @@ -303,8 +312,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage file name should be specified!"); - otaPackageService.saveOtaPackage(firmware); - } + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test public void testSaveFirmwareWithEmptyContentType() { @@ -321,8 +330,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage content type should be specified!"); - otaPackageService.saveOtaPackage(firmware); - } + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test public void testSaveFirmwareWithEmptyData() { @@ -339,8 +348,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage data should be specified!"); - otaPackageService.saveOtaPackage(firmware); - } + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test public void testSaveFirmwareWithInvalidTenant() { @@ -358,8 +367,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage is referencing to non-existent tenant!"); - otaPackageService.saveOtaPackage(firmware); - } + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test public void testSaveFirmwareWithInvalidDeviceProfileId() { @@ -377,7 +386,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage is referencing to non-existent device profile!"); - otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test @@ -395,7 +405,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("OtaPackage checksum should be specified!"); - otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); } @Test @@ -457,7 +468,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { thrown.expect(DataValidationException.class); thrown.expectMessage("Updating otaPackage deviceProfile is prohibited!"); savedFirmware.setDeviceProfileId(null); - otaPackageService.saveOtaPackage(savedFirmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + otaPackageService.saveOtaPackage(savedFirmware, new TestTbMultipartFile(file)); } finally { otaPackageService.deleteOtaPackage(tenantId, savedFirmware.getId()); } @@ -480,7 +492,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { firmware.setChecksum(CHECKSUM); firmware.setData(DATA); firmware.setDataSize(DATA_SIZE); - OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); savedDeviceProfile.setFirmwareId(savedFirmware.getId()); deviceProfileService.saveDeviceProfile(savedDeviceProfile); @@ -581,8 +594,9 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testFindTenantFirmwaresByTenantIdAndHasData() { List firmwares = new ArrayList<>(); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); for (int i = 0; i < 165; i++) { - firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createAndSaveFirmware(tenantId, VERSION + i)))); + firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createAndSaveFirmware(tenantId, VERSION + i), new TestTbMultipartFile(file)))); } OtaPackageInfo firmwareWithUrl = new OtaPackageInfo(); @@ -700,7 +714,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @Test public void testGettingCorrectFileWithOtaData() { - OtaPackage firmware = createFirmware(tenantId, "24687846"); + OtaPackage firmware = createFirmware(tenantId, "24687846", deviceProfileId); File file = otaPackageService.getOtaDataFile(tenantId, firmware.getId()); try { assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, new FileInputStream(file))); @@ -710,7 +724,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { } private OtaPackage createAndSaveFirmware(TenantId tenantId, String version) { - return otaPackageService.saveOtaPackage(createFirmware(tenantId, version, deviceProfileId)); + MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1}); + return otaPackageService.saveOtaPackage(createFirmware(tenantId, version, deviceProfileId), new TestTbMultipartFile(file)); } public static OtaPackage createFirmware( @@ -732,4 +747,37 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { firmware.setDataSize(DATA_SIZE); return firmware; } + + private class TestTbMultipartFile implements TbMultipartFile { + private final MockMultipartFile file; + + private TestTbMultipartFile(MockMultipartFile file) { + this.file = file; + } + + @Override + public Optional getInputStream() { + try { + return Optional.of(file.getInputStream()); + } catch (IOException e) { + return Optional.empty(); + } + } + + @Override + public String getFileName() { + return file.getName(); + } + + @Override + public long getFileSize() { + return file.getSize(); + + } + + @Override + public String getContentType() { + return file.getContentType(); + } + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java index 4a48c56157..f0bfd23bed 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java @@ -21,6 +21,7 @@ import org.junit.Test; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.SpyBean; +import org.springframework.mock.web.MockMultipartFile; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.common.data.Customer; @@ -56,12 +57,12 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.TbMultipartFile; import org.thingsboard.server.dao.tenant.TenantDao; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Set; +import java.io.IOException; +import java.io.InputStream; +import java.util.*; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -584,9 +585,10 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { } private OtaPackage createAndSaveOtaPackageFor(Tenant tenant, DeviceProfile deviceProfile) { + MockMultipartFile file = new MockMultipartFile("filename.txt", new byte[]{1}); return otaPackageService.saveOtaPackage( BaseOtaPackageServiceTest.createFirmware( - tenant.getId(), "2", deviceProfile.getId()) + tenant.getId(), "2", deviceProfile.getId()), new TestTbMultipartFile(file) ); } @@ -695,4 +697,37 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest { tenantProfile.setName("Test tenant profile"); return tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile); } + + private class TestTbMultipartFile implements TbMultipartFile { + private final MockMultipartFile file; + + private TestTbMultipartFile(MockMultipartFile file) { + this.file = file; + } + + @Override + public Optional getInputStream() { + try { + return Optional.of(file.getInputStream()); + } catch (IOException e) { + return Optional.empty(); + } + } + + @Override + public String getFileName() { + return file.getName(); + } + + @Override + public long getFileSize() { + return file.getSize(); + + } + + @Override + public String getContentType() { + return file.getContentType(); + } + } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/sql/OtaPackageServiceSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/sql/OtaPackageServiceSqlTest.java index 549e946eb1..ff31e4f5bb 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/sql/OtaPackageServiceSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/sql/OtaPackageServiceSqlTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.service.sql; + import org.thingsboard.server.dao.service.BaseOtaPackageServiceTest; import org.thingsboard.server.dao.service.DaoSqlTest; From 4d2789b8a170df9b23a2b3248b9174115a036306 Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Wed, 31 Aug 2022 17:17:09 +0300 Subject: [PATCH 11/11] conflicts solved, multipart file added --- .../transport/DefaultTransportApiService.java | 28 ++++++++--------- .../server/dao/ota/OtaPackageService.java | 2 -- .../transport/http/DeviceApiController.java | 31 ++++--------------- .../server/dao/ota/BaseOtaPackageService.java | 18 +++++------ 4 files changed, 28 insertions(+), 51 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index e8da713c7d..b3b617d380 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -101,7 +101,6 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService; import org.thingsboard.server.service.profile.TbDeviceProfileCache; import org.thingsboard.server.service.resource.TbResourceService; -import java.io.File; import java.io.IOException; import java.util.List; import java.util.Optional; @@ -600,17 +599,16 @@ public class DefaultTransportApiService implements TransportApiService { builder.setType(otaPackageInfo.getType().name()); builder.setTitle(otaPackageInfo.getTitle()); builder.setVersion(otaPackageInfo.getVersion()); - File file = otaPackageService.getOtaDataFile(otaPackageInfo.getTenantId(), otaPackageId); - builder.setFileName(file.getAbsolutePath()); + builder.setFileName(otaPackageInfo.getFileName()); builder.setContentType(otaPackageInfo.getContentType()); -// if (!otaPackageDataCache.has(otaPackageId.toString())) { -// OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); -// try { -// otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().readAllBytes()); -// } catch (IOException e) { -// log.error("Failed to cache ota package with id {}",otaPackage.getId(), e); -// } -// } + if (!otaPackageDataCache.has(otaPackageId.toString())) { + OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId); + try { + otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().readAllBytes()); + } catch (IOException e) { + log.error("Failed to cache ota package with id {}",otaPackage.getId(), e); + } + } } } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index 304adba019..a2e1681600 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -16,7 +16,6 @@ package org.thingsboard.server.dao.ota; import com.google.common.util.concurrent.ListenableFuture; -import org.springframework.boot.web.servlet.MultipartConfigFactory; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.id.DeviceProfileId; @@ -27,7 +26,6 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import java.io.File; -import java.io.InputStream; public interface OtaPackageService { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index ca991ff7de..df44a5c61f 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -20,13 +20,10 @@ import com.google.gson.JsonParser; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.core.io.ByteArrayResource; -import org.springframework.core.io.FileSystemResource; -import org.springframework.core.io.Resource; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; @@ -68,8 +65,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseM import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import javax.servlet.http.HttpServletRequest; -import java.io.File; -import java.io.FileInputStream; import java.util.Arrays; import java.util.List; import java.util.UUID; @@ -510,25 +505,15 @@ public class DeviceApiController implements TbTransportService { } @Override - @SneakyThrows public void onSuccess(TransportProtos.GetOtaPackageResponseMsg otaPackageResponseMsg) { if (!TransportProtos.ResponseStatus.SUCCESS.equals(otaPackageResponseMsg.getResponseStatus())) { responseWriter.setResult(new ResponseEntity<>(HttpStatus.NOT_FOUND)); } else if (title.equals(otaPackageResponseMsg.getTitle()) && version.equals(otaPackageResponseMsg.getVersion())) { -// String otaPackageId = new UUID(otaPackageResponseMsg.getOtaPackageIdMSB(), otaPackageResponseMsg.getOtaPackageIdLSB()).toString(); - File file = new File(otaPackageResponseMsg.getFileName()); - Resource resource = null; - if(chuckSize!=0){ - FileInputStream fileInputStream = new FileInputStream(file); - byte[] bytes = new byte[chuckSize]; - fileInputStream.read(bytes, chuck, chuckSize); - resource = new ByteArrayResource(bytes); - } else { - resource= new FileSystemResource(file); - } - ResponseEntity response = ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + getFileName(otaPackageResponseMsg.getFileName())) - .header("x-filename", getFileName(otaPackageResponseMsg.getFileName())) + String otaPackageId = new UUID(otaPackageResponseMsg.getOtaPackageIdMSB(), otaPackageResponseMsg.getOtaPackageIdLSB()).toString(); + ByteArrayResource resource = new ByteArrayResource(transportContext.getOtaPackageDataCache().get(otaPackageId, chuckSize, chuck)); + ResponseEntity response = ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackageResponseMsg.getFileName()) + .header("x-filename", otaPackageResponseMsg.getFileName()) .contentLength(resource.contentLength()) .contentType(parseMediaType(otaPackageResponseMsg.getContentType())) .body(resource); @@ -538,10 +523,6 @@ public class DeviceApiController implements TbTransportService { } } - private String getFileName(String fileName) { - return new File(fileName).getName(); - } - @Override public void onError(Throwable e) { log.warn("Failed to process request", e); @@ -625,7 +606,7 @@ public class DeviceApiController implements TbTransportService { @Override public void onDeviceDeleted(DeviceId deviceId) { UUID sessionId = new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB()); - log.trace("[{}] Received device deleted notification for device with id: {}",sessionId, deviceId); + log.trace("[{}] Received device deleted notification for device with id: {}", sessionId, deviceId); responseWriter.setResult(new ResponseEntity<>("Device was deleted!", HttpStatus.FORBIDDEN)); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index d7db3d30cd..a538fc0359 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -38,9 +38,9 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import javax.transaction.Transactional; -import java.io.*; -import java.util.Collections; -import java.util.List; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.InputStream; import java.util.Optional; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -70,7 +70,7 @@ public class BaseOtaPackageService extends AbstractCachedEntityService optionalStream = file.getInputStream(); - if(optionalStream.isEmpty()){ + if (optionalStream.isEmpty()) { log.error("Failed to get input stream from file {}", file.getFileName()); throw new RuntimeException("Failed to save ota package file"); }