Browse Source

improved fileSaving

pull/7062/head
AnastasiiaKomar 4 years ago
parent
commit
67561bf256
  1. 26
      application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java
  2. 6
      common/cache/pom.xml
  3. 137
      common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java
  4. 188
      common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java
  5. 88
      common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java
  6. 101
      common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java
  7. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java
  8. 32
      dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java
  9. 46
      dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java

26
application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java

@ -21,9 +21,8 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.hibernate.engine.jdbc.BlobProxy; import org.hibernate.engine.jdbc.BlobProxy;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpHeaders; import org.springframework.http.*;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestBody;
@ -41,21 +40,20 @@ import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.ota.util.ChecksumUtil; import org.thingsboard.server.dao.ota.util.ChecksumUtil;
import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.cache.ota.service.FileCacheService;
import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.permission.Resource;
import javax.transaction.Transactional; import javax.transaction.Transactional;
import java.io.InputStream; import java.io.InputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.sql.Blob; import java.sql.Blob;
import java.util.UUID;
import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_ID_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.DEVICE_PROFILE_ID_PARAM_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES; import static org.thingsboard.server.controller.ControllerConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES;
@ -84,14 +82,13 @@ public class OtaPackageController extends BaseController {
public static final String OTA_PACKAGE_ID = "otaPackageId"; public static final String OTA_PACKAGE_ID = "otaPackageId";
public static final String CHECKSUM_ALGORITHM = "checksumAlgorithm"; public static final String CHECKSUM_ALGORITHM = "checksumAlgorithm";
private final FileCacheService fileCacheService;
@ApiOperation(value = "Download OTA Package (downloadOtaPackage)", notes = "Download OTA Package based on the provided OTA Package Id." + TENANT_AUTHORITY_PARAGRAPH) @ApiOperation(value = "Download OTA Package (downloadOtaPackage)", notes = "Download OTA Package based on the provided OTA Package Id." + TENANT_AUTHORITY_PARAGRAPH)
@PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')") @PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')")
@RequestMapping(value = "/otaPackage/{otaPackageId}/download", method = RequestMethod.GET) @RequestMapping(value = "/otaPackage/{otaPackageId}/download", method = RequestMethod.GET)
@ResponseBody @ResponseBody
@Transactional @Transactional
public ResponseEntity<InputStreamResource> downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION) public ResponseEntity<FileSystemResource> downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION)
@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException { @PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException {
checkParameter(OTA_PACKAGE_ID, strOtaPackageId); checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
try { try {
@ -100,18 +97,21 @@ public class OtaPackageController extends BaseController {
if (otaPackage.hasUrl()) { if (otaPackage.hasUrl()) {
return ResponseEntity.badRequest().build(); return ResponseEntity.badRequest().build();
} }
InputStream dataStream = otaPackageService.getOtaDataStream(getTenantId(), otaPackageId); FileSystemResource resource = new FileSystemResource(otaPackageService.getOtaDataFile(getTenantId(), otaPackageId));
InputStreamResource resource = new InputStreamResource(dataStream); String fileName= URLEncoder.encode(otaPackage.getFileName(), StandardCharsets.UTF_8.toString());
return ResponseEntity.ok() return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName()) .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename*=UTF-8''" + fileName)
.header("x-filename", otaPackage.getFileName()) .header("x-filename", fileName)
.contentLength(otaPackage.getDataSize()) .contentLength(otaPackage.getDataSize())
.contentType(parseMediaType(otaPackage.getContentType())) .contentType(new MediaType(parseMediaType(otaPackage.getContentType()), StandardCharsets.UTF_8))
.body(resource); .body(resource);
} catch (Exception e) { } catch (Exception e) {
throw handleException(e); throw handleException(e);
} }
} }
private static String encodeFileName(String fileName) {
return URLEncoder.encode(fileName, StandardCharsets.UTF_8).replace("+", "%20");
}
@ApiOperation(value = "Get OTA Package Info (getOtaPackageInfoById)", @ApiOperation(value = "Get OTA Package Info (getOtaPackageInfoById)",
notes = "Fetch the OTA Package Info object based on the provided OTA Package Id. " + notes = "Fetch the OTA Package Info object based on the provided OTA Package Id. " +

6
common/cache/pom.xml

@ -95,6 +95,12 @@
<artifactId>awaitility</artifactId> <artifactId>awaitility</artifactId>
<scope>test</scope> <scope>test</scope>
</dependency> </dependency>
<dependency>
<groupId>org.hibernate</groupId>
<artifactId>hibernate-core</artifactId>
<version>5.4.32.Final</version>
<scope>test</scope>
</dependency>
</dependencies> </dependencies>
<build> <build>

137
common/cache/src/main/java/org/thingsboard/server/cache/ota/service/BaseFileCacheService.java

@ -0,0 +1,137 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.ota.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.OtaPackageId;
import java.io.*;
import java.sql.Blob;
import java.sql.SQLException;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
@Slf4j
@RequiredArgsConstructor
@Component
public class BaseFileCacheService implements FileCacheService {
@Value("${java.io.tmpdir}/ota/")
private String PATH;
private final TemporaryFileCleaner fileCleaner;
private final ConcurrentMap<OtaPackageId, CountDownLatch> files = new ConcurrentHashMap<>();
public File saveDataTemporaryFile(InputStream inputStream) {
File path = new File(PATH);
try {
File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path);
FileUtils.copyInputStreamToFile(inputStream, tempFile);
return tempFile;
} catch (IOException e) {
log.error("Failed to create temp file", e);
throw new RuntimeException("Failed to create temp file for input stream");
}
}
public File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException {
String fileName = PATH + otaPackageId.getId();
if (exist(fileName)) {
fileCleaner.updateFileUsageStatus(otaPackageId);
return new File(fileName);
}
throw new FileNotFoundException();
}
public File loadToFile(OtaPackageId otaPackageId, Blob data) {
if (otaPackageId == null || data == null) {
log.error("Received null variables: {}", otaPackageId == null ? "otaPackageId" : "data");
throw new NullPointerException();
}
CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data));
String fileName = PATH + otaPackageId.getId();
if (latch.getCount() == 0) {
return new File(fileName);
} else {
try {
latch.await();
return new File(fileName);
} catch (InterruptedException e) {
log.error("Failed to save file {} to file system", fileName, e);
throw new RuntimeException(e);
}
}
}
public void deleteFile(File file) {
try {
if (file.exists()) {
FileUtils.delete(file);
log.info("System file {} was deleted", file.getName());
}
} catch (IOException e) {
log.error("Failed to delete file {}", file.getName(), e);
}
}
public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException {
return new FileInputStream(getOtaDataFile(otaPackageId));
}
public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) {
try {
return new FileInputStream(loadToFile(otaPackageId, data));
} catch (FileNotFoundException e) {
log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e);
throw new RuntimeException(e);
}
}
private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) {
CountDownLatch latch = new CountDownLatch(1);
try {
InputStream binaryStream = data.getBinaryStream();
String fileName = PATH + otaPackageId.getId();
saveAsSystemFile(fileName, binaryStream);
latch.countDown();
return latch;
} catch (SQLException e) {
log.error("Failed to get binary data from otaPackage {}", otaPackageId, e);
throw new RuntimeException("Failed to get binary data from database");
}
}
private void saveAsSystemFile(String fileName, InputStream inputStream) {
try {
File file = new File(fileName);
FileUtils.copyInputStreamToFile(inputStream, file);
} catch (IOException e) {
log.error("Failed to copy stream to system file {}", fileName, e);
throw new RuntimeException("Failed to save file");
}
}
private boolean exist(String name) {
File file = new File(name);
return file.exists();
}
}

188
common/cache/src/main/java/org/thingsboard/server/cache/ota/service/FileCacheService.java

@ -1,183 +1,17 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.ota.service; package org.thingsboard.server.cache.ota.service;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.OtaPackageId;
import javax.annotation.PostConstruct; import java.io.File;
import java.io.*; import java.io.FileNotFoundException;
import java.io.InputStream;
import java.sql.Blob; import java.sql.Blob;
import java.sql.SQLException;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.*;
import java.util.stream.Collectors;
@Slf4j public interface FileCacheService {
@RequiredArgsConstructor void deleteFile(File file);
@Component InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException ;
public class FileCacheService { InputStream loadOtaData(OtaPackageId otaPackageId, Blob data);
@Value("${java.io.tmpdir}/ota/") File loadToFile(OtaPackageId otaPackageId, Blob data);
private String PATH; File saveDataTemporaryFile(InputStream inputStream);
private final static String FILE_PATH_TEMPLATE = "%s"; File getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException;
private final static long TEMPORARY_FILE_INACTIVITY_TIME = 300;//todo increase time }
private final ConcurrentMap<OtaPackageId, CountDownLatch> files = new ConcurrentHashMap<>();
private final ConcurrentMap<OtaPackageId, Long> lastActivityTimes = new ConcurrentHashMap<>();
@PostConstruct
private void createScheduledTaskToClear() {
createTempDirectoryIfNotExist();
cleanDirectoryWithTemporaryFiles();
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 3, 3, TimeUnit.MINUTES); //todo increase time
}
private void createTempDirectoryIfNotExist() {
File directory = new File(PATH);
if(!directory.exists()){
directory.mkdir();
}
}
private void cleanDirectoryWithTemporaryFiles() {
File directory = new File(PATH);
if (directory.isDirectory()) {
try {
FileUtils.cleanDirectory(directory);
log.info("Directory with temporary files cleaned");
} catch (IOException e) {
log.error("Failed to clean directory with temporary files", e);
throw new RuntimeException(e);
}
}
}
private void deleteUnusedTemporaryFiles() {
long currentTime = System.currentTimeMillis();
List<OtaPackageId> toBeDeleted = lastActivityTimes.entrySet()
.stream()
.filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME)
.map(Map.Entry::getKey)
.collect(Collectors.toList());
try {
toBeDeleted.forEach(otaId -> {
deleteFile(otaId.getId().toString());
lastActivityTimes.remove(otaId);
});
} catch (Exception e) {
log.error(",,", e);
}
log.info("Deleted {} unused temporary files", toBeDeleted.size());
}
private void deleteFile(String otaId) {
String fileName = PATH + otaId;
File file = new File(String.format(FILE_PATH_TEMPLATE, fileName));
deleteFile(file);
}
public void deleteFile(File file) {
try {
if (file.exists()) {
FileUtils.delete(file);
log.info("System file {} was deleted", file.getName());
}
} catch (IOException e) {
log.error("Failed to delete file {}", file.getName(), e);
}
}
public InputStream getOtaDataStream(OtaPackageId otaPackageId) throws FileNotFoundException {
String fileName = PATH + otaPackageId.getId();
if (exist(fileName)) {
lastActivityTimes.put(otaPackageId, System.currentTimeMillis());
return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName));
}
throw new FileNotFoundException();
}
private InputStream load(OtaPackageId otaPackageId, Blob data) throws FileNotFoundException {
CountDownLatch latch = files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data));
String fileName = PATH + otaPackageId.getId();
if (latch.getCount() == 0) {
return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName));
} else {
try {
latch.await();
return new FileInputStream(String.format(FILE_PATH_TEMPLATE, fileName));
} catch (InterruptedException e) {
log.error("Failed to save file {} to file system", fileName, e);
throw new RuntimeException(e);
}
}
}
private CountDownLatch processFileSaving(OtaPackageId otaPackageId, Blob data) {
CountDownLatch latch = new CountDownLatch(1);
try {
InputStream binaryStream = data.getBinaryStream();
String fileName =PATH + otaPackageId.getId();
saveAsSystemFile(fileName, binaryStream);
latch.countDown();
return latch;
} catch (SQLException e) {
log.error("Failed to get binary data from otaPackage {}", otaPackageId, e);
throw new RuntimeException("Failed to get binary data from database");
}
}
private void saveAsSystemFile(String fileName, InputStream inputStream) {
try {
File file = new File(String.format(FILE_PATH_TEMPLATE, fileName));
FileUtils.copyInputStreamToFile(inputStream, file);
} catch (IOException e) {
log.error("Failed to copy stream to system file {}", fileName, e);
throw new RuntimeException("Failed to save file");
}
}
private boolean exist(String name) {
File file = new File(String.format(FILE_PATH_TEMPLATE, name));
return file.exists();
}
public InputStream loadOtaData(OtaPackageId otaPackageId, Blob data) {
try {
return load(otaPackageId, data);
} catch (FileNotFoundException e) {
log.error("Failed to upload temporary file {} to system", otaPackageId.getId(), e);
throw new RuntimeException(e);
}
}
public File saveDataTemporaryFile(InputStream inputStream) {
File path = new File(PATH);
try {
File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path);
FileUtils.copyInputStreamToFile(inputStream, tempFile);
return tempFile;
} catch (IOException e) {
log.error("Failed to create temp file", e);
throw new RuntimeException("Failed to create temp file for input stream");
}
}
}

88
common/cache/src/main/java/org/thingsboard/server/cache/ota/service/TemporaryFileCleaner.java

@ -0,0 +1,88 @@
package org.thingsboard.server.cache.ota.service;
import lombok.extern.apachecommons.CommonsLog;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.OtaPackageId;
import javax.annotation.PostConstruct;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;
import java.util.stream.Collectors;
@Slf4j
@Component
public class TemporaryFileCleaner {
@Value("${java.io.tmpdir}/ota/")
private String PATH;
private final static long TEMPORARY_FILE_INACTIVITY_TIME = 900_000;
private final ConcurrentMap<OtaPackageId, Long> lastActivityTimes = new ConcurrentHashMap<>();
public void updateFileUsageStatus(OtaPackageId otaPackageId){
lastActivityTimes.put(otaPackageId, System.currentTimeMillis());
}
@PostConstruct
public void createScheduledTaskToClear() {
createTempDirectoryIfNotExist();
cleanDirectoryWithTemporaryFiles();
ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(this::deleteUnusedTemporaryFiles, 10, 10, TimeUnit.MINUTES);
}
private void createTempDirectoryIfNotExist() {
File directory = new File(PATH);
if (!directory.exists()) {
directory.mkdir();
}
}
private void cleanDirectoryWithTemporaryFiles() {
File directory = new File(PATH);
if (directory.isDirectory()) {
try {
FileUtils.cleanDirectory(directory);
log.info("Directory with temporary files cleaned");
} catch (IOException e) {
log.error("Failed to clean directory with temporary files", e);
throw new RuntimeException(e);
}
}
}
private void deleteUnusedTemporaryFiles() {
long currentTime = System.currentTimeMillis();
List<OtaPackageId> toBeDeleted = lastActivityTimes.entrySet()
.stream()
.filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME)
.map(Map.Entry::getKey)
.collect(Collectors.toList());
try {
toBeDeleted.forEach(otaId -> {
deleteFile(otaId.getId().toString());
lastActivityTimes.remove(otaId);
});
} catch (Exception e) {
log.error("Failed to delete unused files", e);
}
log.info("Deleted {} unused temporary files", toBeDeleted.size());
}
private void deleteFile(String otaId) {
String fileName = PATH + otaId;
File file = new File(fileName);
try {
if (file.exists()) {
FileUtils.delete(file);
log.info("System file {} was deleted", file.getName());
}
} catch (IOException e) {
log.error("Failed to delete file {}", file.getName(), e);
}
}
}

101
common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java

@ -0,0 +1,101 @@
package org.thingsboard.server.cache.ota.service;
import lombok.SneakyThrows;
import org.hibernate.engine.jdbc.BlobProxy;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.thingsboard.server.common.data.id.OtaPackageId;
import java.io.*;
import java.security.MessageDigest;
import java.sql.Blob;
import java.util.Objects;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.*;
class BaseFileCacheServiceTest {
private final static String PATH = "/home/anastasiia/IdeaProjects/thingsboard/common/cache";
private final static String FILE_FILLING = "Hello, testing environment";
private static final int ONE_MEGA_BYTE = 1_000_000;
private final static OtaPackageId OTA_PACKAGE_ID = new OtaPackageId(UUID.randomUUID());
private final static Blob DATA = BlobProxy.generateProxy(FILE_FILLING.getBytes());
File file;
BaseFileCacheService baseFileCacheService = new BaseFileCacheService(new TemporaryFileCleaner());
@BeforeEach
void setUp() {
file = Mockito.mock(File.class);
// fileCacheService = Mockito.mock(FileCacheService.class);
}
@Test
void testNotThrowExceptionsWhenDeleteFile() {
Mockito.when(file.exists()).thenReturn(true);
assertDoesNotThrow(() -> baseFileCacheService.deleteFile(new File("/download/alarm_rule.js")));
}
@Test
void testDataSavingWithNullBlob() {
assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, null));
}
@Test
void testDataSavingWithNullOtaPackageId() {
assertThrows(NullPointerException.class, () -> baseFileCacheService.loadToFile(null, DATA));
}
@Test
@SneakyThrows
void testMultiSavingDataToFile() {
File directory = new File(PATH);
int beginning = Objects.requireNonNull(directory.list()).length;
Thread thread1 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA));
Thread thread2 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA));
thread1.start();
thread2.start();
thread1.join();
thread2.join();
File directory1 = new File(PATH);
int ending = Objects.requireNonNull(directory1.list()).length;
assertEquals(1, ending - beginning);
}
@Test
@SneakyThrows
void testCorrectDataSavingToFile() {
String sha256 = calculateChecksumSHA256(new ByteArrayInputStream(FILE_FILLING.getBytes()));
File file = baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA);
assertEquals(sha256, calculateChecksumSHA256(new FileInputStream(file)));
}
@Test
@SneakyThrows
void testNotThrowExceptionWhenCreatingTemporaryFiles() {
BaseFileCacheService mock = Mockito.mock(BaseFileCacheService.class);
assertDoesNotThrow(() -> mock.saveDataTemporaryFile(DATA.getBinaryStream()));
}
@Test
@SneakyThrows
void testGettingStreamWhenFileNotExist(){
assertThrows(FileNotFoundException.class, ()->baseFileCacheService.getOtaDataStream(OTA_PACKAGE_ID));
}
@SneakyThrows
String calculateChecksumSHA256(InputStream stream) {
MessageDigest md = MessageDigest.getInstance("SHA-256");
byte[] buffer = new byte[ONE_MEGA_BYTE];
int count = 0;
while ((count = stream.read(buffer)) != -1) {
md.update(buffer, 0, count);
}
StringBuilder result = new StringBuilder();
for (byte b : md.digest()) {
result.append(String.format("%02x", b));
}
return result.toString();
}
}

2
common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java

@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import java.io.File;
import java.io.InputStream; import java.io.InputStream;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
@ -54,4 +55,5 @@ public interface OtaPackageService {
long sumDataSizeByTenantId(TenantId tenantId); long sumDataSizeByTenantId(TenantId tenantId);
InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId); InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId);
File getOtaDataFile (TenantId tenantId, OtaPackageId otaPackageId);
} }

32
dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java

@ -20,7 +20,6 @@ import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListenableFuture;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.hibernate.engine.jdbc.BlobProxy; import org.hibernate.engine.jdbc.BlobProxy;
import org.hibernate.exception.ConstraintViolationException; import org.hibernate.exception.ConstraintViolationException;
import org.springframework.cache.Cache; import org.springframework.cache.Cache;
@ -28,7 +27,7 @@ import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.Cacheable; import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.ota.OtaPackageDataCache; import org.thingsboard.server.cache.ota.OtaPackageDataCache;
import org.thingsboard.server.cache.ota.service.FileCacheService; import org.thingsboard.server.cache.ota.service.BaseFileCacheService;
import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.StringUtils;
@ -46,13 +45,10 @@ import org.thingsboard.server.dao.service.PaginatedRemover;
import javax.transaction.Transactional; import javax.transaction.Transactional;
import java.io.*; import java.io.*;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Optional; import java.util.Optional;
import java.util.UUID;
import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE; import static org.thingsboard.server.common.data.CacheConstants.OTA_PACKAGE_CACHE;
import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validateId;
@ -70,7 +66,7 @@ public class BaseOtaPackageService implements OtaPackageService {
private final OtaPackageDataCache otaPackageDataCache; private final OtaPackageDataCache otaPackageDataCache;
private final DataValidator<OtaPackageInfo> otaPackageInfoValidator; private final DataValidator<OtaPackageInfo> otaPackageInfoValidator;
private final DataValidator<OtaPackage> otaPackageValidator; private final DataValidator<OtaPackage> otaPackageValidator;
private final FileCacheService fileCacheService; private final BaseFileCacheService baseFileCacheService;
@Override @Override
public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl) { public OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl) {
@ -101,12 +97,12 @@ public class BaseOtaPackageService implements OtaPackageService {
public OtaPackage saveOtaPackage(OtaPackage otaPackage) { public OtaPackage saveOtaPackage(OtaPackage otaPackage) {
log.trace("Executing saveOtaPackage [{}]", otaPackage); log.trace("Executing saveOtaPackage [{}]", otaPackage);
try { try {
File tempFile= fileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream()); File tempFile= baseFileCacheService.saveDataTemporaryFile(otaPackage.getData().getBinaryStream());
otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize()));
try { try {
otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId); otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId);
}catch (RuntimeException e){ }catch (RuntimeException e){
fileCacheService.deleteFile(tempFile); baseFileCacheService.deleteFile(tempFile);
throw e; throw e;
} }
otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize())); otaPackage.setData(BlobProxy.generateProxy(new FileInputStream(tempFile), otaPackage.getDataSize()));
@ -117,7 +113,7 @@ public class BaseOtaPackageService implements OtaPackageService {
otaPackageDataCache.evict(otaPackageId.toString()); otaPackageDataCache.evict(otaPackageId.toString());
} }
OtaPackage save = otaPackageDao.save(otaPackage.getTenantId(), otaPackage); OtaPackage save = otaPackageDao.save(otaPackage.getTenantId(), otaPackage);
fileCacheService.deleteFile(tempFile); baseFileCacheService.deleteFile(tempFile);
return save; return save;
} catch (FileNotFoundException | SQLException e){ } catch (FileNotFoundException | SQLException e){
log.error("Failed to validate ota package {}",otaPackage.getId(), e); log.error("Failed to validate ota package {}",otaPackage.getId(), e);
@ -268,14 +264,28 @@ public class BaseOtaPackageService implements OtaPackageService {
@Transactional @Transactional
public InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId){ public InputStream getOtaDataStream(TenantId tenantId, OtaPackageId otaPackageId){
try { try {
return fileCacheService.getOtaDataStream(otaPackageId); return baseFileCacheService.getOtaDataStream(otaPackageId);
}catch (FileNotFoundException e){ }catch (FileNotFoundException e){
OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId); OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId);
if (otaPackage == null) { if (otaPackage == null) {
log.error("Can't find otaPackage to download file {}", otaPackage); log.error("Can't find otaPackage to download file {}", otaPackage);
throw new RuntimeException("No such OtaPackageId"); throw new RuntimeException("No such OtaPackageId");
} }
return fileCacheService.loadOtaData(otaPackageId,otaPackage.getData()); return baseFileCacheService.loadOtaData(otaPackageId,otaPackage.getData());
}
}
@Transactional
public File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId){
try {
return baseFileCacheService.getOtaDataFile(otaPackageId);
}catch (FileNotFoundException e){
OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId);
if (otaPackage == null) {
log.error("Can't find otaPackage to download file {}", otaPackage);
throw new RuntimeException("No such OtaPackageId");
}
return baseFileCacheService.loadToFile(otaPackageId,otaPackage.getData());
} }
} }
} }

46
dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java

@ -1,12 +1,12 @@
/** /**
* Copyright © 2016-2022 The Thingsboard Authors * Copyright © 2016-2022 The Thingsboard Authors
* * <p>
* Licensed under the Apache License, Version 2.0 (the "License"); * Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License. * you may not use this file except in compliance with the License.
* You may obtain a copy of the License at * You may obtain a copy of the License at
* * <p>
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* * <p>
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -18,19 +18,10 @@ package org.thingsboard.server.dao.service;
import com.datastax.oss.driver.api.core.uuid.Uuids; import com.datastax.oss.driver.api.core.uuid.Uuids;
import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomStringUtils;
import org.hibernate.engine.jdbc.BlobProxy; import org.hibernate.engine.jdbc.BlobProxy;
import org.junit.After; import org.junit.*;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException; import org.junit.rules.ExpectedException;
import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.*;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm; import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
@ -38,15 +29,20 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.ota.util.ChecksumUtil;
import javax.validation.ValidationException; import javax.validation.ValidationException;
import java.nio.ByteBuffer; import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.sql.Blob; import java.sql.Blob;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE; import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest { public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
@ -697,6 +693,24 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
otaPackageService.saveOtaPackageInfo(firmwareInfo, true); otaPackageService.saveOtaPackageInfo(firmwareInfo, true);
} }
@Test
public void testGettingCorrectStreamWithOtaData() {
OtaPackage firmware = createFirmware(tenantId, "24687846");
InputStream stream = otaPackageService.getOtaDataStream(tenantId, firmware.getId());
assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, stream));
}
@Test
public void testGettingCorrectFileWithOtaData() {
OtaPackage firmware = createFirmware(tenantId, "24687846");
File file = otaPackageService.getOtaDataFile(tenantId, firmware.getId());
try {
assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, new FileInputStream(file)));
} catch (FileNotFoundException e){
throw new RuntimeException(e);
}
}
private OtaPackage createFirmware(TenantId tenantId, String version) { private OtaPackage createFirmware(TenantId tenantId, String version) {
OtaPackage firmware = new OtaPackage(); OtaPackage firmware = new OtaPackage();
firmware.setTenantId(tenantId); firmware.setTenantId(tenantId);

Loading…
Cancel
Save