From 67561bf256b0c99f4caa11c3e050b84bc9105330 Mon Sep 17 00:00:00 2001 From: AnastasiiaKomar Date: Fri, 22 Jul 2022 17:34:15 +0300 Subject: [PATCH] improved fileSaving --- .../controller/OtaPackageController.java | 26 +-- common/cache/pom.xml | 6 + .../ota/service/BaseFileCacheService.java | 137 +++++++++++++ .../cache/ota/service/FileCacheService.java | 188 +----------------- .../ota/service/TemporaryFileCleaner.java | 88 ++++++++ .../ota/service/BaseFileCacheServiceTest.java | 101 ++++++++++ .../server/dao/ota/OtaPackageService.java | 2 + .../server/dao/ota/BaseOtaPackageService.java | 32 ++- .../service/BaseOtaPackageServiceTest.java | 46 +++-- 9 files changed, 409 insertions(+), 217 deletions(-) create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java create mode 100644 common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java create mode 100644 common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java index 719d56510d..062b98028b 100644 --- a/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java +++ b/application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java @@ -21,9 +21,8 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hibernate.engine.jdbc.BlobProxy; -import org.springframework.core.io.InputStreamResource; -import org.springframework.http.HttpHeaders; -import org.springframework.http.ResponseEntity; +import org.springframework.core.io.FileSystemResource; +import org.springframework.http.*; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestBody; @@ -41,21 +40,20 @@ import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.OtaPackageId; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.cache.ota.service.FileCacheService; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import javax.transaction.Transactional; import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.sql.Blob; -import java.util.UUID; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES; @@ -84,14 +82,13 @@ public class OtaPackageController extends BaseController { public static final String OTA_PACKAGE_ID = "otaPackageId"; public static final String CHECKSUM_ALGORITHM = "checksumAlgorithm"; - private final FileCacheService fileCacheService; @ApiOperation(value = "Download OTA Package (downloadOtaPackage)", notes = "Download OTA Package based on the provided OTA Package Id." + TENANT_AUTHORITY_PARAGRAPH) @PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')") @RequestMapping(value = "/otaPackage/{otaPackageId}/download", method = RequestMethod.GET) @ResponseBody @Transactional - public ResponseEntity downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) + public ResponseEntity downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) @PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException { checkParameter(OTA_PACKAGE_ID, strOtaPackageId); try { @@ -100,18 +97,21 @@ public class OtaPackageController extends BaseController { if (otaPackage.hasUrl()) { return ResponseEntity.badRequest().build(); } - InputStream dataStream = otaPackageService.getOtaDataStream(getTenantId(), otaPackageId); - InputStreamResource resource = new InputStreamResource(dataStream); + FileSystemResource resource = new FileSystemResource(otaPackageService.getOtaDataFile(getTenantId(), otaPackageId)); + String fileName= URLEncoder.encode(otaPackage.getFileName(), StandardCharsets.UTF_8.toString()); return ResponseEntity.ok() - .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName()) - .header("x-filename", otaPackage.getFileName()) + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename*=UTF-8''" + fileName) + .header("x-filename", fileName) .contentLength(otaPackage.getDataSize()) - .contentType(parseMediaType(otaPackage.getContentType())) + .contentType(new MediaType(parseMediaType(otaPackage.getContentType()), StandardCharsets.UTF_8)) .body(resource); } catch (Exception e) { throw handleException(e); } } + private static String encodeFileName(String fileName) { + return URLEncoder.encode(fileName, StandardCharsets.UTF_8).replace("+", "%20"); + } @ApiOperation(value = "Get OTA Package Info (getOtaPackageInfoById)", notes = "Fetch the OTA Package Info object based on the provided OTA Package Id. " + diff --git a/common/cache/pom.xml b/common/cache/pom.xml index 4b87f6a78e..c912c4eede 100644 --- a/common/cache/pom.xml +++ b/common/cache/pom.xml @@ -95,6 +95,12 @@ awaitility test + + org.hibernate + hibernate-core + 5.4.32.Final + test + diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java new file mode 100644 index 0000000000..2f2d91a950 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.ota.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.OtaPackageId; + +import java.io.*; +import java.sql.Blob; +import java.sql.SQLException; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CountDownLatch; + +@Slf4j +@RequiredArgsConstructor +@Component +public class BaseFileCacheService implements FileCacheService { + @Value("${java.io.tmpdir}/ota/") + private String PATH; + + private final TemporaryFileCleaner fileCleaner; + private final ConcurrentMap files = new ConcurrentHashMap<>(); + + + public File saveDataTemporaryFile(InputStream inputStream) { + File path = new File(PATH); + try { + File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); + FileUtils.copyInputStreamToFile(inputStream, tempFile); + return tempFile; + } catch (IOException e) { + log.error("Failed to create temp file", e); + throw new RuntimeException("Failed to create temp file for input stream"); + } + } + + public File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException { + String fileName = PATH + otaPackageId.getId(); + if (exist(fileName)) { + fileCleaner.updateFileUsageStatus(otaPackageId); + return new File(fileName); + } + throw new FileNotFoundException(); + } + + public File loadToFile(OtaPackageId otaPackageId, Blob data) { + if (otaPackageId == null || data == null) { + log.error("Received null variables: {}", otaPackageId == null ? "otaPackageId" : "data"); + throw new NullPointerException(); + } + CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data)); + String fileName = PATH + otaPackageId.getId(); + if (latch.getCount() == 0) { + return new File(fileName); + } else { + try { + latch.await(); + return new File(fileName); + } catch (InterruptedException e) { + log.error("Failed to save file {} to file system", fileName, e); + throw new RuntimeException(e); + } + } + } + + public void deleteFile(File file) { + try { + if (file.exists()) { + FileUtils.delete(file); + log.info("System file {} was deleted", file.getName()); + } + } catch (IOException e) { + log.error("Failed to delete file {}", file.getName(), e); + } + } + + public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException { + return new FileInputStream(getOtaDataFile(otaPackageId)); + } + + public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) { + try { + return new FileInputStream(loadToFile(otaPackageId, data)); + } catch (FileNotFoundException e) { + log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e); + throw new RuntimeException(e); + } + } + + private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) { + CountDownLatch latch = new CountDownLatch(1); + try { + InputStream binaryStream = data.getBinaryStream(); + String fileName = PATH + otaPackageId.getId(); + saveAsSystemFile(fileName, binaryStream); + latch.countDown(); + return latch; + } catch (SQLException e) { + log.error("Failed to get binary data from otaPackage {}", otaPackageId, e); + throw new RuntimeException("Failed to get binary data from database"); + } + } + + private void saveAsSystemFile(String fileName, InputStream inputStream) { + try { + File file = new File(fileName); + FileUtils.copyInputStreamToFile(inputStream, file); + } catch (IOException e) { + log.error("Failed to copy stream to system file {}", fileName, e); + throw new RuntimeException("Failed to save file"); + } + } + + private boolean exist(String name) { + File file = new File(name); + return file.exists(); + } +} \ No newline at end of file diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java index 485360b2d0..b022932c4f 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java @@ -1,183 +1,17 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - *

- * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.thingsboard.server.cache.ota.service; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.OtaPackageId; -import javax.annotation.PostConstruct; -import java.io.*; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.sql.Blob; -import java.sql.SQLException; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.*; -import java.util.stream.Collectors; -@Slf4j -@RequiredArgsConstructor -@Component -public class FileCacheService { - @Value("${java.io.tmpdir}/ota/") - private String PATH; - private final static String FILE_PATH_TEMPLATE = "%s"; - private final static long TEMPORARY_FILE_INACTIVITY_TIME = 300;//todo increase time - private final ConcurrentMap files = new ConcurrentHashMap<>(); - private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); - - @PostConstruct - private void createScheduledTaskToClear() { - createTempDirectoryIfNotExist(); - cleanDirectoryWithTemporaryFiles(); - ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 3, 3, TimeUnit.MINUTES); //todo increase time - } - - private void createTempDirectoryIfNotExist() { - File directory = new File(PATH); - if(!directory.exists()){ - directory.mkdir(); - } - } - - private void cleanDirectoryWithTemporaryFiles() { - File directory = new File(PATH); - if (directory.isDirectory()) { - try { - FileUtils.cleanDirectory(directory); - log.info("Directory with temporary files cleaned"); - } catch (IOException e) { - log.error("Failed to clean directory with temporary files", e); - throw new RuntimeException(e); - } - } - } - - private void deleteUnusedTemporaryFiles() { - long currentTime = System.currentTimeMillis(); - List toBeDeleted = lastActivityTimes.entrySet() - .stream() - .filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - try { - toBeDeleted.forEach(otaId -> { - deleteFile(otaId.getId().toString()); - lastActivityTimes.remove(otaId); - }); - } catch (Exception e) { - log.error(",,", e); - } - log.info("Deleted {} unused temporary files", toBeDeleted.size()); - } - - private void deleteFile(String otaId) { - String fileName = PATH + otaId; - File file = new File(String.format(FILE_PATH_TEMPLATE, fileName)); - deleteFile(file); - } - - public void deleteFile(File file) { - try { - if (file.exists()) { - FileUtils.delete(file); - log.info("System file {} was deleted", file.getName()); - } - } catch (IOException e) { - log.error("Failed to delete file {}", file.getName(), e); - } - } - - public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException { - String fileName = PATH + otaPackageId.getId(); - if (exist(fileName)) { - lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); - return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); - } - throw new FileNotFoundException(); - } - - private InputStream load(OtaPackageId otaPackageId, Blob data) throws FileNotFoundException { - CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data)); - String fileName = PATH + otaPackageId.getId(); - if (latch.getCount() == 0) { - return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); - } else { - try { - latch.await(); - return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName)); - } catch (InterruptedException e) { - log.error("Failed to save file {} to file system", fileName, e); - throw new RuntimeException(e); - } - } - } - - private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) { - CountDownLatch latch = new CountDownLatch(1); - try { - InputStream binaryStream = data.getBinaryStream(); - String fileName =PATH + otaPackageId.getId(); - saveAsSystemFile(fileName, binaryStream); - latch.countDown(); - return latch; - } catch (SQLException e) { - log.error("Failed to get binary data from otaPackage {}", otaPackageId, e); - throw new RuntimeException("Failed to get binary data from database"); - } - } - - private void saveAsSystemFile(String fileName, InputStream inputStream) { - try { - File file = new File(String.format(FILE_PATH_TEMPLATE, fileName)); - FileUtils.copyInputStreamToFile(inputStream, file); - } catch (IOException e) { - log.error("Failed to copy stream to system file {}", fileName, e); - throw new RuntimeException("Failed to save file"); - } - } - - private boolean exist(String name) { - File file = new File(String.format(FILE_PATH_TEMPLATE, name)); - return file.exists(); - } - - public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) { - try { - return load(otaPackageId, data); - } catch (FileNotFoundException e) { - log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e); - throw new RuntimeException(e); - } - } - - public File saveDataTemporaryFile(InputStream inputStream) { - File path = new File(PATH); - try { - File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path); - FileUtils.copyInputStreamToFile(inputStream, tempFile); - return tempFile; - } catch (IOException e) { - log.error("Failed to create temp file", e); - throw new RuntimeException("Failed to create temp file for input stream"); - } - } -} \ No newline at end of file +public interface FileCacheService { + void deleteFile(File file); + InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException ; + InputStream loadOtaData(OtaPackageId otaPackageId, Blob data); + File loadToFile(OtaPackageId otaPackageId, Blob data); + File saveDataTemporaryFile(InputStream inputStream); + File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException; +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java new file mode 100644 index 0000000000..dc943c83d9 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java @@ -0,0 +1,88 @@ +package org.thingsboard.server.cache.ota.service; + +import lombok.extern.apachecommons.CommonsLog; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.id.OtaPackageId; + +import javax.annotation.PostConstruct; +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +@Slf4j +@Component +public class TemporaryFileCleaner { + @Value("${java.io.tmpdir}/ota/") + private String PATH; + private final static long TEMPORARY_FILE_INACTIVITY_TIME = 900_000; + private final ConcurrentMap lastActivityTimes = new ConcurrentHashMap<>(); + + public void updateFileUsageStatus(OtaPackageId otaPackageId){ + lastActivityTimes.put(otaPackageId, System.currentTimeMillis()); + } + + @PostConstruct + public void createScheduledTaskToClear() { + createTempDirectoryIfNotExist(); + cleanDirectoryWithTemporaryFiles(); + ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 10, 10, TimeUnit.MINUTES); + } + + private void createTempDirectoryIfNotExist() { + File directory = new File(PATH); + if (!directory.exists()) { + directory.mkdir(); + } + } + + private void cleanDirectoryWithTemporaryFiles() { + File directory = new File(PATH); + if (directory.isDirectory()) { + try { + FileUtils.cleanDirectory(directory); + log.info("Directory with temporary files cleaned"); + } catch (IOException e) { + log.error("Failed to clean directory with temporary files", e); + throw new RuntimeException(e); + } + } + } + + private void deleteUnusedTemporaryFiles() { + long currentTime = System.currentTimeMillis(); + List toBeDeleted = lastActivityTimes.entrySet() + .stream() + .filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); + try { + toBeDeleted.forEach(otaId -> { + deleteFile(otaId.getId().toString()); + lastActivityTimes.remove(otaId); + }); + } catch (Exception e) { + log.error("Failed to delete unused files", e); + } + log.info("Deleted {} unused temporary files", toBeDeleted.size()); + } + + private void deleteFile(String otaId) { + String fileName = PATH + otaId; + File file = new File(fileName); + try { + if (file.exists()) { + FileUtils.delete(file); + log.info("System file {} was deleted", file.getName()); + } + } catch (IOException e) { + log.error("Failed to delete file {}", file.getName(), e); + } + } +} diff --git a/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java new file mode 100644 index 0000000000..20cd1dc47b --- /dev/null +++ b/common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java @@ -0,0 +1,101 @@ +package org.thingsboard.server.cache.ota.service; + +import lombok.SneakyThrows; +import org.hibernate.engine.jdbc.BlobProxy; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.thingsboard.server.common.data.id.OtaPackageId; + +import java.io.*; +import java.security.MessageDigest; +import java.sql.Blob; +import java.util.Objects; +import java.util.UUID; + +import static org.junit.jupiter.api.Assertions.*; + + +class BaseFileCacheServiceTest { + private final static String PATH = "/home/anastasiia/IdeaProjects/thingsboard/common/cache"; + private final static String FILE_FILLING = "Hello, testing environment"; + private static final int ONE_MEGA_BYTE = 1_000_000; + private final static OtaPackageId OTA_PACKAGE_ID = new OtaPackageId(UUID.randomUUID()); + private final static Blob DATA = BlobProxy.generateProxy(FILE_FILLING.getBytes()); + File file; + BaseFileCacheService baseFileCacheService = new BaseFileCacheService(new TemporaryFileCleaner()); + + @BeforeEach + void setUp() { + file = Mockito.mock(File.class); +// fileCacheService = Mockito.mock(FileCacheService.class); + } + + @Test + void testNotThrowExceptionsWhenDeleteFile() { + Mockito.when(file.exists()).thenReturn(true); + assertDoesNotThrow(() -> baseFileCacheService.deleteFile(new File("/download/alarm_rule.js"))); + } + + @Test + void testDataSavingWithNullBlob() { + assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, null)); + } + + @Test + void testDataSavingWithNullOtaPackageId() { + assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(null, DATA)); + } + + @Test + @SneakyThrows + void testMultiSavingDataToFile() { + File directory = new File(PATH); + int beginning = Objects.requireNonNull(directory.list()).length; + Thread thread1 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA)); + Thread thread2 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA)); + thread1.start(); + thread2.start(); + thread1.join(); + thread2.join(); + File directory1 = new File(PATH); + int ending = Objects.requireNonNull(directory1.list()).length; + assertEquals(1, ending - beginning); + } + + @Test + @SneakyThrows + void testCorrectDataSavingToFile() { + String sha256 = calculateChecksumSHA256(new ByteArrayInputStream(FILE_FILLING.getBytes())); + File file = baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA); + assertEquals(sha256, calculateChecksumSHA256(new FileInputStream(file))); + } + + @Test + @SneakyThrows + void testNotThrowExceptionWhenCreatingTemporaryFiles() { + BaseFileCacheService mock = Mockito.mock(BaseFileCacheService.class); + assertDoesNotThrow(() -> mock.saveDataTemporaryFile(DATA.getBinaryStream())); + } + + @Test + @SneakyThrows + void testGettingStreamWhenFileNotExist(){ + assertThrows(FileNotFoundException.class, ()->baseFileCacheService.getOtaDataStream(OTA_PACKAGE_ID)); + } + + @SneakyThrows + String calculateChecksumSHA256(InputStream stream) { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] buffer = new byte[ONE_MEGA_BYTE]; + int count = 0; + while ((count = stream.read(buffer)) != -1) { + md.update(buffer, 0, count); + } + StringBuilder result = new StringBuilder(); + for (byte b : md.digest()) { + result.append(String.format("%02x", b)); + } + return result.toString(); + } +} \ No newline at end of file diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java index 8ce5e64066..c0c5b64a1f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import java.io.File; import java.io.InputStream; import java.nio.ByteBuffer; @@ -54,4 +55,5 @@ public interface OtaPackageService { long sumDataSizeByTenantId(TenantId tenantId); InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId); + File getOtaDataFile (TenantId tenantId, OtaPackageId otaPackageId); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index ffa6906cd0..435479932a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -20,7 +20,6 @@ import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.io.FileUtils; import org.hibernate.engine.jdbc.BlobProxy; import org.hibernate.exception.ConstraintViolationException; import org.springframework.cache.Cache; @@ -28,7 +27,7 @@ import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.ota.OtaPackageDataCache; -import org.thingsboard.server.cache.ota.service.FileCacheService; +import org.thingsboard.server.cache.ota.service.BaseFileCacheService; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.StringUtils; @@ -46,13 +45,10 @@ import org.thingsboard.server.dao.service.PaginatedRemover; import javax.transaction.Transactional; import java.io.*; import java.nio.ByteBuffer; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.UUID; import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; @@ -70,7 +66,7 @@ public class BaseOtaPackageService implements OtaPackageService { private final OtaPackageDataCache otaPackageDataCache; private final DataValidator otaPackageInfoValidator; private final DataValidator otaPackageValidator; - private final FileCacheService fileCacheService; + private final BaseFileCacheService baseFileCacheService; @Override public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl) { @@ -101,12 +97,12 @@ public class BaseOtaPackageService implements OtaPackageService { public OtaPackage saveOtaPackage(OtaPackage otaPackage) { log.trace("Executing saveOtaPackage [{}]", otaPackage); try { - File tempFile= fileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream()); + File tempFile= baseFileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream()); otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); try { otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); }catch (RuntimeException e){ - fileCacheService.deleteFile(tempFile); + baseFileCacheService.deleteFile(tempFile); throw e; } otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); @@ -117,7 +113,7 @@ public class BaseOtaPackageService implements OtaPackageService { otaPackageDataCache.evict(otaPackageId.toString()); } OtaPackage save = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); - fileCacheService.deleteFile(tempFile); + baseFileCacheService.deleteFile(tempFile); return save; } catch (FileNotFoundException | SQLException e){ log.error("Failed to validate ota package {}",otaPackage.getId(), e); @@ -268,14 +264,28 @@ public class BaseOtaPackageService implements OtaPackageService { @Transactional public InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId){ try { - return fileCacheService.getOtaDataStream(otaPackageId); + return baseFileCacheService.getOtaDataStream(otaPackageId); }catch (FileNotFoundException e){ OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); if (otaPackage == null) { log.error("Can't find otaPackage to download file {}", otaPackage); throw new RuntimeException("No such OtaPackageId"); } - return fileCacheService.loadOtaData(otaPackageId,otaPackage.getData()); + return baseFileCacheService.loadOtaData(otaPackageId,otaPackage.getData()); + } + } + + @Transactional + public File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId){ + try { + return baseFileCacheService.getOtaDataFile(otaPackageId); + }catch (FileNotFoundException e){ + OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); + if (otaPackage == null) { + log.error("Can't find otaPackage to download file {}", otaPackage); + throw new RuntimeException("No such OtaPackageId"); + } + return baseFileCacheService.loadToFile(otaPackageId,otaPackage.getData()); } } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java index 006392bb45..1e0b0340c0 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java @@ -1,12 +1,12 @@ /** * Copyright © 2016-2022 The Thingsboard Authors - * + *

* Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,19 +18,10 @@ package org.thingsboard.server.dao.service; import com.datastax.oss.driver.api.core.uuid.Uuids; import org.apache.commons.lang3.RandomStringUtils; import org.hibernate.engine.jdbc.BlobProxy; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; +import org.junit.*; import org.junit.rules.ExpectedException; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.Device; -import org.thingsboard.server.common.data.DeviceProfile; -import org.thingsboard.server.common.data.OtaPackage; -import org.thingsboard.server.common.data.OtaPackageInfo; -import org.thingsboard.server.common.data.Tenant; -import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.*; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; @@ -38,15 +29,20 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.util.ChecksumUtil; import javax.validation.ValidationException; -import java.nio.ByteBuffer; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.InputStream; import java.sql.Blob; import java.util.ArrayList; import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { @@ -697,6 +693,24 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { otaPackageService.saveOtaPackageInfo(firmwareInfo, true); } + @Test + public void testGettingCorrectStreamWithOtaData() { + OtaPackage firmware = createFirmware(tenantId, "24687846"); + InputStream stream = otaPackageService.getOtaDataStream(tenantId, firmware.getId()); + assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, stream)); + } + + @Test + public void testGettingCorrectFileWithOtaData() { + OtaPackage firmware = createFirmware(tenantId, "24687846"); + File file = otaPackageService.getOtaDataFile(tenantId, firmware.getId()); + try { + assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, new FileInputStream(file))); + } catch (FileNotFoundException e){ + throw new RuntimeException(e); + } + } + private OtaPackage createFirmware(TenantId tenantId, String version) { OtaPackage firmware = new OtaPackage(); firmware.setTenantId(tenantId);