Browse Source

Merge branch 'firmware_update' of github.com:Natiik/thingsboard into feature/firmware-footprint

pull/7222/head
Andrii Shvaika 4 years ago
parent
commit
ba4a767033
  1. 25
      application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java
  2. 1
      application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java
  3. 8
      application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java
  4. 37
      application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java
  5. 3
      application/src/main/java/org/thingsboard/server/service/entitiy/ota/TbOtaPackageService.java
  6. 42
      application/src/main/java/org/thingsboard/server/service/ota/TbMultipartFileImp.java
  7. 15
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  8. 3
      application/src/main/resources/thingsboard.yml
  9. 44
      application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java
  10. 5
      common/cache/pom.xml
  11. 100
      common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java
  12. 29
      common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java
  13. 122
      common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java
  14. 93
      common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java
  15. 9
      common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java
  16. 15
      common/dao-api/src/main/java/org/thingsboard/server/dao/ota/TbMultipartFile.java
  17. 11
      common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java
  18. 23
      common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java
  19. 2
      common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java
  20. 40
      dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java
  21. 84
      dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java
  22. 92
      dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java
  23. 19
      dao/src/main/java/org/thingsboard/server/dao/service/validator/AbstractHasOtaPackageValidator.java
  24. 14
      dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java
  25. 83
      dao/src/test/java/org/thingsboard/server/dao/ota/util/ChecksumUtilTest.java
  26. 4
      dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java
  27. 47
      dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java
  28. 61
      dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java
  29. 104
      dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java
  30. 45
      dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java
  31. 1
      dao/src/test/java/org/thingsboard/server/dao/service/sql/OtaPackageServiceSqlTest.java

25
application/src/main/java/org/thingsboard/server/controller/OtaPackageController.java

@ -19,7 +19,8 @@ import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.ByteArrayResource;
import org.apache.commons.lang3.StringUtils;
import org.springframework.core.io.FileSystemResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.ResponseEntity;
import org.springframework.security.access.prepost.PreAuthorize;
@ -42,11 +43,13 @@ import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.ota.util.ChecksumUtil;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.ota.TbOtaPackageService;
import org.thingsboard.server.service.security.permission.Operation;
import org.thingsboard.server.service.security.permission.Resource;
import javax.transaction.Transactional;
import java.io.IOException;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
@ -84,8 +87,9 @@ public class OtaPackageController extends BaseController {
@PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')")
@RequestMapping(value = "/otaPackage/{otaPackageId}/download", method = RequestMethod.GET)
@ResponseBody
public ResponseEntity<org.springframework.core.io.Resource> downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION)
@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException {
@Transactional
public ResponseEntity<FileSystemResource> downloadOtaPackage(@ApiParam(value = OTA_PACKAGE_ID_PARAM_DESCRIPTION)
@PathVariable(OTA_PACKAGE_ID) String strOtaPackageId) throws ThingsboardException {
checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
try {
OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId));
@ -94,12 +98,11 @@ public class OtaPackageController extends BaseController {
if (otaPackage.hasUrl()) {
return ResponseEntity.badRequest().build();
}
ByteArrayResource resource = new ByteArrayResource(otaPackage.getData().array());
FileSystemResource resource = new FileSystemResource(otaPackageService.getOtaDataFile(getTenantId(), otaPackageId));
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + otaPackage.getFileName())
.header("x-filename", otaPackage.getFileName())
.contentLength(resource.contentLength())
.contentLength(otaPackage.getDataSize())
.contentType(parseMediaType(otaPackage.getContentType()))
.body(resource);
} catch (Exception e) {
@ -176,21 +179,19 @@ public class OtaPackageController extends BaseController {
@ApiParam(value = "OTA Package checksum algorithm.", allowableValues = OTA_PACKAGE_CHECKSUM_ALGORITHM_ALLOWABLE_VALUES)
@RequestParam(CHECKSUM_ALGORITHM) String checksumAlgorithmStr,
@ApiParam(value = "OTA Package data.")
@RequestPart MultipartFile file) throws ThingsboardException, IOException {
@RequestPart MultipartFile file) throws ThingsboardException {
checkParameter(OTA_PACKAGE_ID, strOtaPackageId);
checkParameter(CHECKSUM_ALGORITHM, checksumAlgorithmStr);
OtaPackageId otaPackageId = new OtaPackageId(toUUID(strOtaPackageId));
OtaPackageInfo otaPackageInfo = checkOtaPackageInfoId(otaPackageId, Operation.READ);
OtaPackageInfo info = checkOtaPackageInfoId(otaPackageId, Operation.READ);
ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.valueOf(checksumAlgorithmStr.toUpperCase());
byte[] data = file.getBytes();
return tbOtaPackageService.saveOtaPackageData(otaPackageInfo, checksum, checksumAlgorithm,
data, file.getOriginalFilename(), file.getContentType(), getCurrentUser());
return tbOtaPackageService.saveOtaPackageData(info, checksum, checksumAlgorithm, file, getCurrentUser());
}
@ApiOperation(value = "Get OTA Package Infos (getOtaPackages)",
notes = "Returns a page of OTA Package Info objects owned by tenant. " +
PAGE_DATA_PARAMETERS + OTA_PACKAGE_INFO_DESCRIPTION + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH,
produces = APPLICATION_JSON_VALUE)
produces = "application/json")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')")
@RequestMapping(value = "/otaPackages", method = RequestMethod.GET)
@ResponseBody

1
application/src/main/java/org/thingsboard/server/service/action/EntityActionService.java

@ -269,5 +269,4 @@ public class EntityActionService {
entityNode.put(kvEntry.getKey(), kvEntry.getValueAsString());
}
}
}

8
application/src/main/java/org/thingsboard/server/service/edge/rpc/constructor/OtaPackageMsgConstructor.java

@ -24,6 +24,8 @@ import org.thingsboard.server.gen.edge.v1.OtaPackageUpdateMsg;
import org.thingsboard.server.gen.edge.v1.UpdateMsgType;
import org.thingsboard.server.queue.util.TbCoreComponent;
import java.io.IOException;
@Component
@TbCoreComponent
public class OtaPackageMsgConstructor {
@ -65,7 +67,11 @@ public class OtaPackageMsgConstructor {
builder.setDataSize(otaPackage.getDataSize());
}
if (otaPackage.getData() != null) {
builder.setData(ByteString.copyFrom(otaPackage.getData().array()));
try {
builder.setData(ByteString.copyFrom(otaPackage.getData().readAllBytes()));
} catch (IOException e){
throw new RuntimeException(e);
}
}
return builder.build();
}

37
application/src/main/java/org/thingsboard/server/service/entitiy/ota/DefaultTbOtaPackageService.java

@ -1,12 +1,12 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -18,22 +18,25 @@ package org.thingsboard.server.service.entitiy.ota;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.OtaPackage;
import org.springframework.web.multipart.MultipartFile;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.ota.util.ChecksumUtil;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.ota.TbMultipartFileImp;
import java.nio.ByteBuffer;
import java.io.IOException;
@Service
@TbCoreComponent
@ -64,12 +67,12 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen
@Override
public OtaPackageInfo saveOtaPackageData(OtaPackageInfo otaPackageInfo, String checksum, ChecksumAlgorithm checksumAlgorithm,
byte[] data, String filename, String contentType, User user) throws ThingsboardException {
MultipartFile file, User user) {
TenantId tenantId = otaPackageInfo.getTenantId();
OtaPackageId otaPackageId = otaPackageInfo.getId();
try {
if (StringUtils.isEmpty(checksum)) {
checksum = otaPackageService.generateChecksum(checksumAlgorithm, ByteBuffer.wrap(data));
checksum = ChecksumUtil.generateChecksum(checksumAlgorithm, file.getInputStream());
}
OtaPackage otaPackage = new OtaPackage(otaPackageId);
otaPackage.setCreatedTime(otaPackageInfo.getCreatedTime());
@ -82,14 +85,18 @@ public class DefaultTbOtaPackageService extends AbstractTbEntityService implemen
otaPackage.setAdditionalInfo(otaPackageInfo.getAdditionalInfo());
otaPackage.setChecksumAlgorithm(checksumAlgorithm);
otaPackage.setChecksum(checksum);
otaPackage.setFileName(filename);
otaPackage.setContentType(contentType);
otaPackage.setData(ByteBuffer.wrap(data));
otaPackage.setDataSize((long) data.length);
OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage);
otaPackage.setFileName(file.getOriginalFilename());
otaPackage.setContentType(file.getContentType());
otaPackage.setData(file.getInputStream());
otaPackage.setDataSize(file.getSize());
OtaPackageInfo savedOtaPackage = otaPackageService.saveOtaPackage(otaPackage, new TbMultipartFileImp(file));
notificationEntityService.notifyCreateOrUpdateOrDelete(tenantId, null, savedOtaPackage.getId(),
savedOtaPackage, user, ActionType.UPDATED, true, null);
return savedOtaPackage;
} catch (IOException e){
notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.OTA_PACKAGE), ActionType.UPDATED,
user, e, otaPackageId.toString());
throw new RuntimeException(e);
} catch (Exception e) {
notificationEntityService.logEntityAction(tenantId, emptyId(EntityType.OTA_PACKAGE), ActionType.UPDATED,
user, e, otaPackageId.toString());

3
application/src/main/java/org/thingsboard/server/service/entitiy/ota/TbOtaPackageService.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.service.entitiy.ota;
import org.springframework.web.multipart.MultipartFile;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.SaveOtaPackageInfoRequest;
import org.thingsboard.server.common.data.User;
@ -26,7 +27,7 @@ public interface TbOtaPackageService {
OtaPackageInfo save(SaveOtaPackageInfoRequest saveOtaPackageInfoRequest, User user) throws ThingsboardException;
OtaPackageInfo saveOtaPackageData(OtaPackageInfo otaPackageInfo, String checksum, ChecksumAlgorithm checksumAlgorithm,
byte[] data, String filename, String contentType, User user) throws ThingsboardException;
MultipartFile file, User user) throws ThingsboardException;
void delete(OtaPackageInfo otaPackageInfo, User user) throws ThingsboardException;

42
application/src/main/java/org/thingsboard/server/service/ota/TbMultipartFileImp.java

@ -0,0 +1,42 @@
package org.thingsboard.server.service.ota;
import lombok.RequiredArgsConstructor;
import org.springframework.web.multipart.MultipartFile;
import org.thingsboard.server.dao.ota.TbMultipartFile;
import javax.validation.constraints.NotNull;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
@RequiredArgsConstructor
public class TbMultipartFileImp implements TbMultipartFile {
@NotNull
private final MultipartFile file;
@Override
public Optional<InputStream> getInputStream() {
try {
return Optional.of(file.getInputStream());
} catch (IOException e) {
return Optional.empty();
}
}
@Override
public String getFileName() {
return file.getName();
}
@Override
public long getFileSize() {
return file.getSize();
}
@Override
public String getContentType() {
return file.getContentType();
}
}

15
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -1,12 +1,12 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -101,6 +101,7 @@ import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.resource.TbResourceService;
import java.io.IOException;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
@ -602,7 +603,11 @@ public class DefaultTransportApiService implements TransportApiService {
builder.setContentType(otaPackageInfo.getContentType());
if (!otaPackageDataCache.has(otaPackageId.toString())) {
OtaPackage otaPackage = otaPackageService.findOtaPackageById(tenantId, otaPackageId);
otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().array());
try {
otaPackageDataCache.put(otaPackageId.toString(), otaPackage.getData().readAllBytes());
} catch (IOException e) {
log.error("Failed to cache ota package with id {}",otaPackage.getId(), e);
}
}
}
}

3
application/src/main/resources/thingsboard.yml

@ -1148,3 +1148,6 @@ management:
exposure:
# Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics).
include: '${METRICS_ENDPOINTS_EXPOSE:info}'
files:
temporary_files_directory: "${TEMPORARY_FILES_DIRECTORY: ${java.io.tmpdir}}"

44
application/src/test/java/org/thingsboard/server/service/sync/ie/BaseExportImportServiceTest.java

@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.node.TextNode;
import org.junit.After;
import org.junit.Before;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.mock.web.MockMultipartFile;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.rule.engine.debug.TbMsgGeneratorNode;
import org.thingsboard.rule.engine.debug.TbMsgGeneratorNodeConfiguration;
@ -71,17 +72,23 @@ import org.thingsboard.server.dao.device.DeviceProfileService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.ota.TbMultipartFile;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.rule.RuleChainService;
import org.thingsboard.server.dao.service.BaseOtaPackageServiceTest;
import org.thingsboard.server.dao.tenant.TenantService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.Optional;
import java.util.UUID;
import static org.assertj.core.api.Assertions.assertThat;
@ -171,8 +178,9 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest
otaPackage.setChecksumAlgorithm(ChecksumAlgorithm.SHA256);
otaPackage.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a");
otaPackage.setDataSize(1L);
otaPackage.setData(ByteBuffer.wrap(new byte[]{(int) 1}));
return otaPackageService.saveOtaPackage(otaPackage);
otaPackage.setData(new ByteArrayInputStream(new byte[]{1}));
MockMultipartFile file = new MockMultipartFile("filename.txt", new byte[]{1});
return otaPackageService.saveOtaPackage(otaPackage, new TestTbMultipartFile(file));
}
protected void checkImportedDeviceData(Device initialDevice, Device importedDevice) {
@ -449,4 +457,36 @@ public abstract class BaseExportImportServiceTest extends AbstractControllerTest
return new SecurityUser(user, true, new UserPrincipal(UserPrincipal.Type.USER_NAME, user.getEmail()));
}
private class TestTbMultipartFile implements TbMultipartFile {
private final MockMultipartFile file;
private TestTbMultipartFile(MockMultipartFile file) {
this.file = file;
}
@Override
public Optional<InputStream> getInputStream() {
try {
return Optional.of(file.getInputStream());
} catch (IOException e) {
return Optional.empty();
}
}
@Override
public String getFileName() {
return file.getName();
}
@Override
public long getFileSize() {
return file.getSize();
}
@Override
public String getContentType() {
return file.getContentType();
}
}
}

5
common/cache/pom.xml

@ -44,6 +44,11 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>org.springframework.data</groupId>
<artifactId>spring-data-redis</artifactId>

100
common/cache/src/main/java/org/thingsboard/server/cache/ota/files/BaseFileCacheService.java

@ -0,0 +1,100 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.ota.files;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.OtaPackageId;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@Slf4j
@RequiredArgsConstructor
@Component
public class BaseFileCacheService implements FileCacheService {
@Value("${files.temporary_files_directory}/ota/")
private String PATH;
private final static String FILE_NAME_TEMPLATE = "%s.tmp";
private final TemporaryFileCleaner fileCleaner;
private final ConcurrentMap<OtaPackageId, Boolean> files = new ConcurrentHashMap<>();
@Override
public File saveDataTemporaryFile(InputStream inputStream) {
File path = new File(PATH);
try {
File tempFile = File.createTempFile(UUID.randomUUID().toString(), ".tmp", path);
FileUtils.copyInputStreamToFile(inputStream, tempFile);
return tempFile;
} catch (IOException e) {
log.error("Failed to create temp file", e);
throw new RuntimeException("Failed to create temp file for input stream");
}
}
@Override
public Optional<File> getOtaDataFile(OtaPackageId otaPackageId) {
String fileName = PATH + String.format(FILE_NAME_TEMPLATE,otaPackageId.getId().toString());
if (exist(fileName)) {
fileCleaner.updateFileUsageStatus(otaPackageId);
return Optional.of(new File(fileName));
}
return Optional.empty();
}
@Override
public File loadToFile(OtaPackageId otaPackageId, InputStream data) {
if (otaPackageId == null || data == null) {
log.error("Received null variables: {}", otaPackageId == null ? "otaPackageId" : "data");
throw new RuntimeException("Input values can not be null");
}
files.computeIfAbsent(otaPackageId, ota -> processFileSaving(ota, data));
String fileName = PATH + String.format(FILE_NAME_TEMPLATE,otaPackageId.getId().toString());
return new File(fileName);
}
private Boolean processFileSaving(OtaPackageId otaPackageId, InputStream data) {
String fileName = PATH + String.format(FILE_NAME_TEMPLATE,otaPackageId.getId().toString());
saveAsSystemFile(fileName, data);
fileCleaner.updateFileUsageStatus(otaPackageId);
return true;
}
private void saveAsSystemFile(String fileName, InputStream inputStream) {
try {
File file = new File(fileName);
FileUtils.copyInputStreamToFile(inputStream, file);
} catch (IOException e) {
log.error("Failed to copy stream to system file {}", fileName, e);
throw new RuntimeException("Failed to save file");
}
}
private boolean exist(String name) {
File file = new File(name);
return file.exists();
}
}

29
common/cache/src/main/java/org/thingsboard/server/cache/ota/files/FileCacheService.java

@ -0,0 +1,29 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.ota.files;
import org.thingsboard.server.common.data.id.OtaPackageId;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.Optional;
public interface FileCacheService {
File loadToFile(OtaPackageId otaPackageId, InputStream data);
File saveDataTemporaryFile(InputStream inputStream);
Optional<File> getOtaDataFile(OtaPackageId otaPackageId) throws FileNotFoundException;
}

122
common/cache/src/main/java/org/thingsboard/server/cache/ota/files/TemporaryFileCleaner.java

@ -0,0 +1,122 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.ota.files;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import org.thingsboard.server.common.data.id.OtaPackageId;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.stream.Collectors;
@Slf4j
@Component
public class TemporaryFileCleaner {
@Value("${files.temporary_files_directory}/ota/")
private String PATH;
private final static String FILE_NAME_TEMPLATE = "%s%s.tmp";
private final static long TEMPORARY_FILE_INACTIVITY_TIME = 900_000;
private final ConcurrentMap<OtaPackageId, Long> lastActivityTimes = new ConcurrentHashMap<>();
public void updateFileUsageStatus(OtaPackageId otaPackageId) {
lastActivityTimes.put(otaPackageId, System.currentTimeMillis());
}
@EventListener(ApplicationReadyEvent.class)
public void cleanDirectoryWithTemporaryFiles() {
createTempDirectoryIfNotExist();
cleanDirectory();
log.info("Directory {} with temporary ota files cleaned", PATH);
}
private void createTempDirectoryIfNotExist() {
File directory = new File(PATH);
if (!directory.exists()) {
try{
FileUtils.forceMkdir(directory);
} catch(IOException e){
log.error("Failed to create directory for temporary files ", e);
}
}
}
private void cleanDirectory() {
File directory = new File(PATH);
if (directory.isDirectory()) {
File[] files = directory.listFiles();
if (files == null) return;
Arrays.stream(files).forEach(
file -> {
try {
FileUtils.delete(file);
} catch (Exception e) {
log.error("Failed to delete file {}", file.getName(), e);
}
}
);
}
}
@Scheduled(fixedDelay = 600_000)
private void deleteUnusedTemporaryFiles() {
long currentTime = System.currentTimeMillis();
List<OtaPackageId> toBeDeleted = lastActivityTimes.entrySet()
.stream()
.filter(entry -> currentTime - entry.getValue() > TEMPORARY_FILE_INACTIVITY_TIME)
.map(Map.Entry::getKey)
.collect(Collectors.toList());
try {
toBeDeleted.forEach(otaId -> {
deleteFile(otaId.getId().toString());
lastActivityTimes.remove(otaId);
});
} catch (Exception e) {
log.error("Failed to delete unused files", e);
}
log.info("Deleted {} unused temporary files", toBeDeleted.size());
}
private synchronized void deleteFile(String otaId) {
String fileName = String.format(FILE_NAME_TEMPLATE, PATH, otaId);
File file = new File(fileName);
try (FileChannel channel = FileChannel.open(Path.of(URI.create(file.getPath())), StandardOpenOption.APPEND)) {
FileLock lock = channel.lock();
if (file.exists()) {
FileUtils.delete(file);
log.info("System file {} was deleted", file.getName());
}
lock.release();
} catch (IOException e) {
log.error("Failed to delete file {}", file.getName(), e);
}
}
}

93
common/cache/src/test/java/org/thingsboard/server/cache/ota/service/BaseFileCacheServiceTest.java

@ -0,0 +1,93 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.ota.service;
import lombok.SneakyThrows;
import org.junit.jupiter.api.Test;
import org.thingsboard.server.cache.ota.files.BaseFileCacheService;
import org.thingsboard.server.cache.ota.files.TemporaryFileCleaner;
import org.thingsboard.server.common.data.id.OtaPackageId;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.security.MessageDigest;
import java.util.Objects;
import java.util.UUID;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
class BaseFileCacheServiceTest {
private final static String PATH = "/home/anastasiia/IdeaProjects/thingsboard/common/cache";
private final static String FILE_FILLING = "Hello, testing environment";
private static final int ONE_MEGA_BYTE = 1_000_000;
private final static OtaPackageId OTA_PACKAGE_ID = new OtaPackageId(UUID.randomUUID());
private final static InputStream DATA = new ByteArrayInputStream(FILE_FILLING.getBytes());
BaseFileCacheService baseFileCacheService = new BaseFileCacheService(new TemporaryFileCleaner());
@Test
void testDataSavingWithNullInputStream() {
assertThrows(RuntimeException.class, () -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, null));
}
@Test
void testDataSavingWithNullOtaPackageId() {
assertThrows(RuntimeException.class, () -> baseFileCacheService.loadToFile(null, DATA));
}
@Test
@SneakyThrows
void testMultiSavingDataToFile() {
File directory = new File(PATH);
int beginning = Objects.requireNonNull(directory.list()).length;
Thread thread1 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA));
Thread thread2 = new Thread(() -> baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA));
thread1.start();
thread2.start();
thread1.join();
thread2.join();
File directory1 = new File(PATH);
int ending = Objects.requireNonNull(directory1.list()).length;
assertEquals(1, ending - beginning);
}
@Test
@SneakyThrows
void testCorrectDataSavingToFile() {
String sha256 = calculateChecksumSHA256(new ByteArrayInputStream(FILE_FILLING.getBytes()));
File file = baseFileCacheService.loadToFile(OTA_PACKAGE_ID, DATA);
assertEquals(sha256, calculateChecksumSHA256(new FileInputStream(file)));
}
@SneakyThrows
String calculateChecksumSHA256(InputStream stream) {
MessageDigest md = MessageDigest.getInstance("SHA-256");
byte[] buffer = new byte[ONE_MEGA_BYTE];
int count = 0;
while ((count = stream.read(buffer)) != -1) {
md.update(buffer, 0, count);
}
StringBuilder result = new StringBuilder();
for (byte b : md.digest()) {
result.append(String.format("%02x", b));
}
return result.toString();
}
}

9
common/dao-api/src/main/java/org/thingsboard/server/dao/ota/OtaPackageService.java

@ -21,20 +21,17 @@ import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import java.nio.ByteBuffer;
import java.io.File;
public interface OtaPackageService {
OtaPackageInfo saveOtaPackageInfo(OtaPackageInfo otaPackageInfo, boolean isUrl);
OtaPackage saveOtaPackage(OtaPackage otaPackage);
String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data);
OtaPackage saveOtaPackage(OtaPackage otaPackage, TbMultipartFile file);
OtaPackage findOtaPackageById(TenantId tenantId, OtaPackageId otaPackageId);
@ -51,4 +48,6 @@ public interface OtaPackageService {
void deleteOtaPackagesByTenantId(TenantId tenantId);
long sumDataSizeByTenantId(TenantId tenantId);
File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId);
}

15
common/dao-api/src/main/java/org/thingsboard/server/dao/ota/TbMultipartFile.java

@ -0,0 +1,15 @@
package org.thingsboard.server.dao.ota;
import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
public interface TbMultipartFile {
Optional<InputStream> getInputStream();
String getFileName();
long getFileSize();
String getContentType();
}

11
common/data/src/main/java/org/thingsboard/server/common/data/OtaPackage.java

@ -15,23 +15,28 @@
*/
package org.thingsboard.server.common.data;
import com.fasterxml.jackson.annotation.JsonAutoDetect;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.thingsboard.server.common.data.id.OtaPackageId;
import java.nio.ByteBuffer;
import java.io.InputStream;
@ApiModel
@Data
@EqualsAndHashCode(callSuper = true)
@JsonIgnoreProperties(value = { "data" })
public class OtaPackage extends OtaPackageInfo {
private static final long serialVersionUID = 3091601761339422546L;
@ApiModelProperty(position = 16, value = "OTA Package data.", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private transient ByteBuffer data;
@ApiModelProperty(position = 16, value = "OTA Package data.", readOnly = true)
private transient InputStream data;
public OtaPackage() {
super();

23
common/data/src/main/java/org/thingsboard/server/common/data/ota/ChecksumAlgorithm.java

@ -15,12 +15,21 @@
*/
package org.thingsboard.server.common.data.ota;
import lombok.Getter;
public enum ChecksumAlgorithm {
MD5,
SHA256,
SHA384,
SHA512,
CRC32,
MURMUR3_32,
MURMUR3_128
MD5("MD5"),
SHA256("SHA-256"),
SHA384("SHA-384"),
SHA512("SHA-512"),
CRC32("CRC32"),
MURMUR3_32("MURMUR3-32"),
MURMUR3_128("MURMUR3_128");
@Getter
final String name;
ChecksumAlgorithm(String name) {
this.name = name;
}
}

2
common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java

@ -606,7 +606,7 @@ public class DeviceApiController implements TbTransportService {
@Override
public void onDeviceDeleted(DeviceId deviceId) {
UUID sessionId = new UUID(sessionInfo.getSessionIdMSB(), sessionInfo.getSessionIdLSB());
log.trace("[{}] Received device deleted notification for device with id: {}",sessionId, deviceId);
log.trace("[{}] Received device deleted notification for device with id: {}", sessionId, deviceId);
responseWriter.setResult(new ResponseEntity<>("Device was deleted!", HttpStatus.FORBIDDEN));
}

40
dao/src/main/java/org/thingsboard/server/dao/model/sql/OtaPackageEntity.java

@ -18,8 +18,14 @@ package org.thingsboard.server.dao.model.sql;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.SneakyThrows;
import org.apache.commons.io.FileUtils;
import org.aspectj.util.FileUtil;
import org.hibernate.annotations.Type;
import org.hibernate.annotations.TypeDef;
import org.hibernate.engine.jdbc.BlobProxy;
import org.postgresql.largeobject.BlobInputStream;
import org.springframework.beans.factory.annotation.Value;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId;
@ -31,30 +37,13 @@ import org.thingsboard.server.dao.model.ModelConstants;
import org.thingsboard.server.dao.model.SearchTextEntity;
import org.thingsboard.server.dao.util.mapping.JsonStringType;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.EnumType;
import javax.persistence.Enumerated;
import javax.persistence.Lob;
import javax.persistence.Table;
import java.nio.ByteBuffer;
import javax.persistence.*;
import java.io.File;
import java.io.FileInputStream;
import java.sql.Blob;
import java.util.UUID;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_CHECKSUM_ALGORITHM_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_CHECKSUM_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_CONTENT_TYPE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_DATA_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_DATA_SIZE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_DEVICE_PROFILE_ID_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TAG_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TENANT_ID_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TILE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_TYPE_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_URL_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.OTA_PACKAGE_VERSION_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.SEARCH_TEXT_PROPERTY;
import static org.thingsboard.server.dao.model.ModelConstants.*;
@Data
@EqualsAndHashCode(callSuper = true)
@ -100,7 +89,7 @@ public class OtaPackageEntity extends BaseSqlEntity<OtaPackage> implements Searc
@Lob
@Column(name = OTA_PACKAGE_DATA_COLUMN, columnDefinition = "BINARY")
private byte[] data;
private Blob data;
@Column(name = OTA_PACKAGE_DATA_SIZE_COLUMN)
private Long dataSize;
@ -132,7 +121,7 @@ public class OtaPackageEntity extends BaseSqlEntity<OtaPackage> implements Searc
this.contentType = otaPackage.getContentType();
this.checksumAlgorithm = otaPackage.getChecksumAlgorithm();
this.checksum = otaPackage.getChecksum();
this.data = otaPackage.getData().array();
this.data = BlobProxy.generateProxy(otaPackage.getData(), otaPackage.getDataSize());
this.dataSize = otaPackage.getDataSize();
this.additionalInfo = otaPackage.getAdditionalInfo();
}
@ -148,6 +137,7 @@ public class OtaPackageEntity extends BaseSqlEntity<OtaPackage> implements Searc
}
@Override
@SneakyThrows
public OtaPackage toData() {
OtaPackage otaPackage = new OtaPackage(new OtaPackageId(id));
otaPackage.setCreatedTime(createdTime);
@ -166,7 +156,7 @@ public class OtaPackageEntity extends BaseSqlEntity<OtaPackage> implements Searc
otaPackage.setChecksum(checksum);
otaPackage.setDataSize(dataSize);
if (data != null) {
otaPackage.setData(ByteBuffer.wrap(data));
otaPackage.setData(data.getBinaryStream());
otaPackage.setHasData(true);
}
otaPackage.setAdditionalInfo(additionalInfo);

84
dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java

@ -1,12 +1,12 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@ -15,8 +15,6 @@
*/
package org.thingsboard.server.dao.ota;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -24,13 +22,13 @@ import org.hibernate.exception.ConstraintViolationException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.server.cache.ota.OtaPackageDataCache;
import org.thingsboard.server.cache.ota.files.BaseFileCacheService;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.OtaPackageId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import org.thingsboard.server.common.data.ota.OtaPackageType;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@ -39,7 +37,10 @@ import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.service.PaginatedRemover;
import java.nio.ByteBuffer;
import javax.transaction.Transactional;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.InputStream;
import java.util.Optional;
import static org.thingsboard.server.dao.service.Validator.validateId;
@ -57,6 +58,7 @@ public class BaseOtaPackageService extends AbstractCachedEntityService<OtaPackag
private final OtaPackageDataCache otaPackageDataCache;
private final DataValidator<OtaPackageInfo> otaPackageInfoValidator;
private final DataValidator<OtaPackage> otaPackageValidator;
private final BaseFileCacheService baseFileCacheService;
@TransactionalEventListener(classes = OtaPackageCacheEvictEvent.class)
@Override
@ -93,20 +95,24 @@ public class BaseOtaPackageService extends AbstractCachedEntityService<OtaPackag
}
@Override
public OtaPackage saveOtaPackage(OtaPackage otaPackage) {
public OtaPackage saveOtaPackage(OtaPackage otaPackage, TbMultipartFile file) {
log.trace("Executing saveOtaPackage [{}]", otaPackage);
otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId);
OtaPackageId otaPackageId = otaPackage.getId();
try {
var result = otaPackageDao.save(otaPackage.getTenantId(), otaPackage);
otaPackageValidator.validate(otaPackage, OtaPackageInfo::getTenantId);
Optional<InputStream> optionalStream = file.getInputStream();
if (optionalStream.isEmpty()) {
log.error("Failed to get input stream from file {}", file.getFileName());
throw new RuntimeException("Failed to save ota package file");
}
BufferedInputStream stream = new BufferedInputStream(optionalStream.get());
stream.mark(0);
otaPackage.setData(stream);
OtaPackageId otaPackageId = otaPackage.getId();
if (otaPackageId != null) {
publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId));
}
return result;
return otaPackageDao.save(otaPackage.getTenantId(), otaPackage);
} catch (Exception t) {
if (otaPackageId != null) {
handleEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId));
}
ConstraintViolationException e = extractConstraintViolationException(t).orElse(null);
if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("ota_package_tenant_title_version_unq_key")) {
throw new DataValidationException("OtaPackage with such title and version already exists!");
@ -116,37 +122,6 @@ public class BaseOtaPackageService extends AbstractCachedEntityService<OtaPackag
}
}
@Override
public String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data) {
if (data == null || !data.hasArray() || data.array().length == 0) {
throw new DataValidationException("OtaPackage data should be specified!");
}
return getHashFunction(checksumAlgorithm).hashBytes(data.array()).toString();
}
@SuppressWarnings("deprecation")
private HashFunction getHashFunction(ChecksumAlgorithm checksumAlgorithm) {
switch (checksumAlgorithm) {
case MD5:
return Hashing.md5();
case SHA256:
return Hashing.sha256();
case SHA384:
return Hashing.sha384();
case SHA512:
return Hashing.sha512();
case CRC32:
return Hashing.crc32();
case MURMUR3_32:
return Hashing.murmur3_32();
case MURMUR3_128:
return Hashing.murmur3_128();
default:
throw new DataValidationException("Unknown checksum algorithm!");
}
}
@Override
public OtaPackage findOtaPackageById(TenantId tenantId, OtaPackageId otaPackageId) {
log.trace("Executing findOtaPackageById [{}]", otaPackageId);
@ -234,4 +209,19 @@ public class BaseOtaPackageService extends AbstractCachedEntityService<OtaPackag
}
};
@Override
@Transactional
public File getOtaDataFile(TenantId tenantId, OtaPackageId otaPackageId) {
validateId(otaPackageId, INCORRECT_OTA_PACKAGE_ID + otaPackageId);
Optional<File> otaDataFile = baseFileCacheService.getOtaDataFile(otaPackageId);
if (otaDataFile.isPresent()) {
return otaDataFile.get();
}
OtaPackage otaPackage = findOtaPackageById(tenantId, otaPackageId);
if (otaPackage == null) {
log.error("Can't find otaPackage to download file {}", otaPackageId);
throw new RuntimeException("No such OtaPackageId");
}
return baseFileCacheService.loadToFile(otaPackageId, otaPackage.getData());
}
}

92
dao/src/main/java/org/thingsboard/server/dao/ota/util/ChecksumUtil.java

@ -0,0 +1,92 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.ota.util;
import com.datastax.oss.driver.shaded.guava.common.io.ByteStreams;
import com.google.common.hash.Funnels;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.data.ota.ChecksumAlgorithm;
import java.io.IOException;
import java.io.InputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Base64;
import java.util.zip.CRC32;
@Slf4j
public class ChecksumUtil {
private static final int ONE_MEGA_BYTE = 1_000_000;
public static String generateChecksum(ChecksumAlgorithm checksumAlgorithm, InputStream fileData) {
try {
switch (checksumAlgorithm) {
case CRC32:
return checksumCRC32(fileData);
case MURMUR3_128:
return checksumMurmur3_128(fileData);
case MURMUR3_32:
return checksumMurmur3_32(fileData);
default:
MessageDigest md = MessageDigest.getInstance(checksumAlgorithm.getName());
return checksum(fileData, md);
}
} catch (NoSuchAlgorithmException e) {
log.error("No such checksum algorithm {}", checksumAlgorithm, e);
throw new RuntimeException(e);
} catch (Exception e) {
log.error("Failed to calculate checksum", e);
throw new RuntimeException(e);
}
}
private static String checksum(InputStream inputStream, MessageDigest md) throws IOException {
byte[] buffer = new byte[ONE_MEGA_BYTE];
int count = 0;
while ((count = inputStream.read(buffer)) != -1) {
md.update(buffer, 0, count);
}
StringBuilder result = new StringBuilder();
for (byte b : md.digest()) {
result.append(String.format("%02x", b));
}
return result.toString();
}
private static String checksumCRC32(InputStream inputStream) throws IOException {
CRC32 crc = new CRC32();
byte[] buffer = new byte[ONE_MEGA_BYTE];
int count = 0;
while ((count = inputStream.read(buffer)) != -1) {
crc.update(buffer, 0, count);
}
return Long.toHexString(crc.getValue());
}
private static String checksumMurmur3_32(InputStream stream) throws IOException {
Hasher hasher = Hashing.murmur3_32().newHasher();
com.google.common.io.ByteStreams.copy(stream, Funnels.asOutputStream(hasher));
return hasher.hash().toString();
}
private static String checksumMurmur3_128(InputStream stream) throws IOException {
Hasher hasher = Hashing.murmur3_128().newHasher();
ByteStreams.copy(stream, Funnels.asOutputStream(hasher));
return hasher.hash().toString();
}
}

19
dao/src/main/java/org/thingsboard/server/dao/service/validator/AbstractHasOtaPackageValidator.java

@ -20,6 +20,7 @@ import org.springframework.context.annotation.Lazy;
import org.thingsboard.server.common.data.BaseData;
import org.thingsboard.server.common.data.HasOtaPackage;
import org.thingsboard.server.common.data.OtaPackage;
import org.thingsboard.server.common.data.OtaPackageInfo;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.ota.OtaPackageType;
@ -35,29 +36,29 @@ public abstract class AbstractHasOtaPackageValidator<D extends BaseData<?>> exte
protected <T extends HasOtaPackage> void validateOtaPackage(TenantId tenantId, T entity, DeviceProfileId deviceProfileId) {
if (entity.getFirmwareId() != null) {
OtaPackage firmware = otaPackageService.findOtaPackageById(tenantId, entity.getFirmwareId());
OtaPackageInfo firmware = otaPackageService.findOtaPackageInfoById(tenantId, entity.getFirmwareId());
validateOtaPackage(tenantId, OtaPackageType.FIRMWARE, deviceProfileId, firmware);
}
if (entity.getSoftwareId() != null) {
OtaPackage software = otaPackageService.findOtaPackageById(tenantId, entity.getSoftwareId());
OtaPackageInfo software = otaPackageService.findOtaPackageInfoById(tenantId, entity.getSoftwareId());
validateOtaPackage(tenantId, OtaPackageType.SOFTWARE, deviceProfileId, software);
}
}
private void validateOtaPackage(TenantId tenantId, OtaPackageType type, DeviceProfileId deviceProfileId, OtaPackage otaPackage) {
if (otaPackage == null) {
private void validateOtaPackage(TenantId tenantId, OtaPackageType type, DeviceProfileId deviceProfileId, OtaPackageInfo otaPackageInfo) {
if (otaPackageInfo == null) {
throw new DataValidationException(prepareMsg("Can't assign non-existent %s!", type));
}
if (!otaPackage.getTenantId().equals(tenantId)) {
if (!otaPackageInfo.getTenantId().equals(tenantId)) {
throw new DataValidationException(prepareMsg("Can't assign %s from different tenant!", type));
}
if (!otaPackage.getType().equals(type)) {
throw new DataValidationException(prepareMsg("Can't assign %s with type: " + otaPackage.getType(), type));
if (!otaPackageInfo.getType().equals(type)) {
throw new DataValidationException(prepareMsg("Can't assign %s with type: " + otaPackageInfo.getType(), type));
}
if (otaPackage.getData() == null && !otaPackage.hasUrl()) {
if (otaPackageInfo.getDataSize() == 0 && !otaPackageInfo.hasUrl()) {
throw new DataValidationException(prepareMsg("Can't assign %s with empty data!", type));
}
if (!otaPackage.getDeviceProfileId().equals(deviceProfileId)) {
if (!otaPackageInfo.getDeviceProfileId().equals(deviceProfileId)) {
throw new DataValidationException(prepareMsg("Can't assign %s with different deviceProfile!", type));
}
}

14
dao/src/main/java/org/thingsboard/server/dao/service/validator/OtaPackageDataValidator.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.dao.service.validator;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
@ -24,21 +25,18 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.ota.OtaPackageDao;
import org.thingsboard.server.dao.ota.OtaPackageService;
import org.thingsboard.server.dao.ota.util.ChecksumUtil;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import static org.thingsboard.server.common.data.EntityType.OTA_PACKAGE;
@Component
@Slf4j
public class OtaPackageDataValidator extends BaseOtaPackageDataValidator<OtaPackage> {
@Autowired
private OtaPackageDao otaPackageDao;
@Autowired
@Lazy
private OtaPackageService otaPackageService;
@Autowired
@Lazy
private TbTenantProfileCache tenantProfileCache;
@ -70,11 +68,7 @@ public class OtaPackageDataValidator extends BaseOtaPackageDataValidator<OtaPack
if (StringUtils.isEmpty(otaPackage.getChecksum())) {
throw new DataValidationException("OtaPackage checksum should be specified!");
}
String currentChecksum;
currentChecksum = otaPackageService.generateChecksum(otaPackage.getChecksumAlgorithm(), otaPackage.getData());
String currentChecksum = ChecksumUtil.generateChecksum(otaPackage.getChecksumAlgorithm(), otaPackage.getData());
if (!currentChecksum.equals(otaPackage.getChecksum())) {
throw new DataValidationException("Wrong otaPackage file!");
}

83
dao/src/test/java/org/thingsboard/server/dao/ota/util/ChecksumUtilTest.java

@ -0,0 +1,83 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.ota.util;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.io.ByteArrayInputStream;
import java.util.Arrays;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.thingsboard.server.common.data.ota.ChecksumAlgorithm.*;
class ChecksumUtilTest {
private static final int SIZE = 1_050_000;
ByteArrayInputStream inputStream;
@BeforeEach
void setUp() {
char[] chars = new char[SIZE];
Arrays.fill(chars, 'f');
String s = new String(chars);
inputStream = new ByteArrayInputStream(s.getBytes());
}
@Test
void testSha256Checksum() {
String generateChecksum = ChecksumUtil.generateChecksum(SHA256, inputStream);
assertEquals("b23c6c2faa06fe3f9a47b86914d08a77c9db9ce9b87fc8b6b2758db756159b2e", generateChecksum);
}
@Test
void testMd5Checksum() {
String generateChecksum = ChecksumUtil.generateChecksum(MD5, inputStream);
assertEquals("352b53342cc1ef2a21c480656ef6ffe6", generateChecksum);
}
@Test
void testSha384Checksum() {
String generateChecksum = ChecksumUtil.generateChecksum(SHA384, inputStream);
assertEquals("bd04277a8fc6ace52123d6d6214d2612ccdec802f361974198c44fd3df857b79f2343f010943340aeb7c51321a2d32e9", generateChecksum);
}
@Test
void testSha512Checksum() {
String generateChecksum = ChecksumUtil.generateChecksum(SHA512, inputStream);
assertEquals("2efb6e7ef97eca294e8cc4bf731615622199fba59c2b5ed4f9a56c1f17be3522abac4f5d9fa0f95f39ff10a59c28597a263697b2d794e6686d260ffaa078da7e",
generateChecksum);
}
@Test
void testCrc32Checksum() {
String generateChecksum = ChecksumUtil.generateChecksum(CRC32, inputStream);
assertEquals("4b8f2fda", generateChecksum);
}
@Test
void testMurmur3_32Checksum() {
assertEquals("3d45a1dd", ChecksumUtil.generateChecksum(MURMUR3_32, inputStream));
assertDoesNotThrow(() -> ChecksumUtil.generateChecksum(MURMUR3_32, inputStream));
}
@Test
void testMurmur3_128Checksum() {
assertEquals("9dbf0ffe5f6ecbb5edf207717e7870b7", ChecksumUtil.generateChecksum(MURMUR3_128, inputStream));
assertDoesNotThrow(() -> ChecksumUtil.generateChecksum(MURMUR3_128, inputStream));
}
}

4
dao/src/test/java/org/thingsboard/server/dao/service/AbstractServiceTest.java

@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.uuid.Uuids;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.commons.lang3.RandomStringUtils;
import org.hibernate.engine.jdbc.BlobProxy;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
@ -79,6 +80,7 @@ import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Comparator;
@ -281,7 +283,7 @@ public abstract class AbstractServiceTest {
firmware.setContentType("text/plain");
firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256);
firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a");
firmware.setData(ByteBuffer.wrap(new byte[]{1}));
firmware.setData(new ByteArrayInputStream(new byte[]{1}));
firmware.setDataSize(1L);
return firmware;
}

47
dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java

@ -19,11 +19,13 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import org.hibernate.engine.jdbc.BlobProxy;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.mock.web.MockMultipartFile;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
@ -36,11 +38,16 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.ota.TbMultipartFile;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@ -111,9 +118,10 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest {
firmware.setContentType("text/plain");
firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256);
firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a");
firmware.setData(ByteBuffer.wrap(new byte[]{1}));
firmware.setData(new ByteArrayInputStream(new byte[]{1}));
firmware.setDataSize(1L);
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile("test.txt", new byte[]{1});
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
deviceProfile.setFirmwareId(savedFirmware.getId());
@ -259,7 +267,8 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest {
DeviceProfile deviceProfile = this.createDeviceProfile(tenantId, "Device Profile");
deviceProfile = deviceProfileService.saveDeviceProfile(deviceProfile);
OtaPackage otaPackage = constructDefaultOtaPackage(tenantId, deviceProfile.getId());
otaPackage = otaPackageService.saveOtaPackage(otaPackage);
MockMultipartFile file = new MockMultipartFile("filename.txt", new byte[]{1});
otaPackage = otaPackageService.saveOtaPackage(otaPackage, new TestTbMultipartFile(file));
assertThat(deviceProfileService.findDeviceProfileById(tenantId, deviceProfile.getId())).isNotNull();
assertThat(otaPackageService.findOtaPackageById(tenantId, otaPackage.getId())).isNotNull();
@ -370,4 +379,36 @@ public abstract class BaseDeviceProfileServiceTest extends AbstractServiceTest {
Assert.assertEquals(1, pageData.getTotalElements());
}
private class TestTbMultipartFile implements TbMultipartFile {
private final MockMultipartFile file;
private TestTbMultipartFile(MockMultipartFile file) {
this.file = file;
}
@Override
public Optional<InputStream> getInputStream() {
try {
return Optional.of(file.getInputStream());
} catch (IOException e) {
return Optional.empty();
}
}
@Override
public String getFileName() {
return file.getName();
}
@Override
public long getFileSize() {
return file.getSize();
}
@Override
public String getContentType() {
return file.getContentType();
}
}
}

61
dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java

@ -23,6 +23,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.springframework.mock.web.MockMultipartFile;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceInfo;
@ -40,11 +41,15 @@ import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.data.security.DeviceCredentialsType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.ota.TbMultipartFile;
import java.nio.ByteBuffer;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID;
@ -200,9 +205,10 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
firmware.setContentType("text/plain");
firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256);
firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a");
firmware.setData(ByteBuffer.wrap(new byte[]{1}));
firmware.setData(new ByteArrayInputStream(new byte[]{1}));
firmware.setDataSize(1L);
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile("test.txt", new byte[]{1});
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
savedDevice.setFirmwareId(savedFirmware.getId());
@ -235,9 +241,11 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
firmware.setContentType("text/plain");
firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256);
firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a");
firmware.setData(ByteBuffer.wrap(new byte[]{1}));
firmware.setData(new ByteArrayInputStream(new byte[]{1}));
firmware.setDataSize(1L);
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile("test.txt", new byte[]{1});
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
savedDevice.setFirmwareId(savedFirmware.getId());
@ -846,8 +854,8 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
deviceInfosWithLabel.stream()
.anyMatch(
d -> d.getId().equals(savedDevice.getId())
&& d.getTenantId().equals(tenantId)
&& d.getLabel().equals(savedDevice.getLabel())
&& d.getTenantId().equals(tenantId)
&& d.getLabel().equals(savedDevice.getLabel())
)
);
@ -905,9 +913,9 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
deviceInfosWithLabel.stream()
.anyMatch(
d -> d.getId().equals(savedDevice.getId())
&& d.getTenantId().equals(tenantId)
&& d.getDeviceProfileName().equals(savedDevice.getType())
&& d.getLabel().equals(savedDevice.getLabel())
&& d.getTenantId().equals(tenantId)
&& d.getDeviceProfileName().equals(savedDevice.getType())
&& d.getLabel().equals(savedDevice.getLabel())
)
);
@ -973,4 +981,37 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest {
)
);
}
private class TestTbMultipartFile implements TbMultipartFile {
private final MockMultipartFile file;
private TestTbMultipartFile(MockMultipartFile file) {
this.file = file;
}
@Override
public Optional<InputStream> getInputStream() {
try {
return Optional.of(file.getInputStream());
} catch (IOException e) {
return Optional.empty();
}
}
@Override
public String getFileName() {
return file.getName();
}
@Override
public long getFileSize() {
return file.getSize();
}
@Override
public String getContentType() {
return file.getContentType();
}
}
}

104
dao/src/test/java/org/thingsboard/server/dao/service/BaseOtaPackageServiceTest.java

@ -23,6 +23,7 @@ import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.springframework.mock.web.MockMultipartFile;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
@ -37,14 +38,20 @@ import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.ota.TbMultipartFile;
import org.thingsboard.server.dao.ota.util.ChecksumUtil;
import javax.validation.ValidationException;
import java.io.*;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.InstanceOfAssertFactories.FILE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.thingsboard.server.common.data.ota.OtaPackageType.FIRMWARE;
public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
@ -56,7 +63,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
private static final ChecksumAlgorithm CHECKSUM_ALGORITHM = ChecksumAlgorithm.SHA256;
private static final String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a";
private static final long DATA_SIZE = 1L;
private static final ByteBuffer DATA = ByteBuffer.wrap(new byte[]{(int) DATA_SIZE});
private static final InputStream DATA = new ByteArrayInputStream(new byte[]{1});
private static final String URL = "http://firmware.test.org";
private final IdComparator<OtaPackageInfo> idComparator = new IdComparator<>();
@ -137,7 +144,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
firmware.setChecksum(CHECKSUM);
firmware.setData(DATA);
firmware.setDataSize(DATA_SIZE);
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
Assert.assertNotNull(savedFirmware);
Assert.assertNotNull(savedFirmware.getId());
@ -149,7 +157,7 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
Assert.assertEquals(firmware.getData(), savedFirmware.getData());
savedFirmware.setAdditionalInfo(JacksonUtil.newObjectNode());
otaPackageService.saveOtaPackage(savedFirmware);
otaPackageService.saveOtaPackage(savedFirmware, new TestTbMultipartFile(file));
OtaPackage foundFirmware = otaPackageService.findOtaPackageById(tenantId, savedFirmware.getId());
Assert.assertEquals(foundFirmware.getTitle(), savedFirmware.getTitle());
@ -216,7 +224,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
firmware.setData(DATA);
firmware.setDataSize(DATA_SIZE);
otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
savedFirmwareInfo = otaPackageService.findOtaPackageInfoById(tenantId, savedFirmwareInfo.getId());
savedFirmwareInfo.setAdditionalInfo(JacksonUtil.newObjectNode());
@ -246,7 +255,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage should be assigned to tenant!");
otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
}
@Test
@ -264,7 +274,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("Type should be specified!");
otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
}
@Test
@ -282,7 +293,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage title should be specified!");
otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
}
@Test
@ -300,8 +312,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage file name should be specified!");
otaPackageService.saveOtaPackage(firmware);
}
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); }
@Test
public void testSaveFirmwareWithEmptyContentType() {
@ -318,8 +330,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage content type should be specified!");
otaPackageService.saveOtaPackage(firmware);
}
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); }
@Test
public void testSaveFirmwareWithEmptyData() {
@ -336,8 +348,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage data should be specified!");
otaPackageService.saveOtaPackage(firmware);
}
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); }
@Test
public void testSaveFirmwareWithInvalidTenant() {
@ -355,8 +367,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage is referencing to non-existent tenant!");
otaPackageService.saveOtaPackage(firmware);
}
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file)); }
@Test
public void testSaveFirmwareWithInvalidDeviceProfileId() {
@ -374,7 +386,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage is referencing to non-existent device profile!");
otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
}
@Test
@ -392,7 +405,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("OtaPackage checksum should be specified!");
otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
}
@Test
@ -454,7 +468,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
thrown.expect(DataValidationException.class);
thrown.expectMessage("Updating otaPackage deviceProfile is prohibited!");
savedFirmware.setDeviceProfileId(null);
otaPackageService.saveOtaPackage(savedFirmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
otaPackageService.saveOtaPackage(savedFirmware, new TestTbMultipartFile(file));
} finally {
otaPackageService.deleteOtaPackage(tenantId, savedFirmware.getId());
}
@ -477,7 +492,8 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
firmware.setChecksum(CHECKSUM);
firmware.setData(DATA);
firmware.setDataSize(DATA_SIZE);
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware);
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
OtaPackage savedFirmware = otaPackageService.saveOtaPackage(firmware, new TestTbMultipartFile(file));
savedDeviceProfile.setFirmwareId(savedFirmware.getId());
deviceProfileService.saveDeviceProfile(savedDeviceProfile);
@ -578,8 +594,9 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
@Test
public void testFindTenantFirmwaresByTenantIdAndHasData() {
List<OtaPackageInfo> firmwares = new ArrayList<>();
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
for (int i = 0; i < 165; i++) {
firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createAndSaveFirmware(tenantId, VERSION + i))));
firmwares.add(new OtaPackageInfo(otaPackageService.saveOtaPackage(createAndSaveFirmware(tenantId, VERSION + i), new TestTbMultipartFile(file))));
}
OtaPackageInfo firmwareWithUrl = new OtaPackageInfo();
@ -695,8 +712,20 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
otaPackageService.saveOtaPackageInfo(firmwareInfo, true);
}
@Test
public void testGettingCorrectFileWithOtaData() {
OtaPackage firmware = createFirmware(tenantId, "24687846", deviceProfileId);
File file = otaPackageService.getOtaDataFile(tenantId, firmware.getId());
try {
assertEquals(firmware.getChecksum(), ChecksumUtil.generateChecksum(CHECKSUM_ALGORITHM, new FileInputStream(file)));
} catch (FileNotFoundException e){
throw new RuntimeException(e);
}
}
private OtaPackage createAndSaveFirmware(TenantId tenantId, String version) {
return otaPackageService.saveOtaPackage(createFirmware(tenantId, version, deviceProfileId));
MockMultipartFile file = new MockMultipartFile(FILE_NAME, new byte[]{1});
return otaPackageService.saveOtaPackage(createFirmware(tenantId, version, deviceProfileId), new TestTbMultipartFile(file));
}
public static OtaPackage createFirmware(
@ -718,4 +747,37 @@ public abstract class BaseOtaPackageServiceTest extends AbstractServiceTest {
firmware.setDataSize(DATA_SIZE);
return firmware;
}
private class TestTbMultipartFile implements TbMultipartFile {
private final MockMultipartFile file;
private TestTbMultipartFile(MockMultipartFile file) {
this.file = file;
}
@Override
public Optional<InputStream> getInputStream() {
try {
return Optional.of(file.getInputStream());
} catch (IOException e) {
return Optional.empty();
}
}
@Override
public String getFileName() {
return file.getName();
}
@Override
public long getFileSize() {
return file.getSize();
}
@Override
public String getContentType() {
return file.getContentType();
}
}
}

45
dao/src/test/java/org/thingsboard/server/dao/service/BaseTenantServiceTest.java

@ -21,6 +21,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.SpyBean;
import org.springframework.mock.web.MockMultipartFile;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.Customer;
@ -56,12 +57,12 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon
import org.thingsboard.server.common.data.tenant.profile.TenantProfileData;
import org.thingsboard.server.common.data.widget.WidgetsBundle;
import org.thingsboard.server.dao.exception.DataValidationException;
import org.thingsboard.server.dao.ota.TbMultipartFile;
import org.thingsboard.server.dao.tenant.TenantDao;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
@ -584,9 +585,10 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest {
}
private OtaPackage createAndSaveOtaPackageFor(Tenant tenant, DeviceProfile deviceProfile) {
MockMultipartFile file = new MockMultipartFile("filename.txt", new byte[]{1});
return otaPackageService.saveOtaPackage(
BaseOtaPackageServiceTest.createFirmware(
tenant.getId(), "2", deviceProfile.getId())
tenant.getId(), "2", deviceProfile.getId()), new TestTbMultipartFile(file)
);
}
@ -695,4 +697,37 @@ public abstract class BaseTenantServiceTest extends AbstractServiceTest {
tenantProfile.setName("Test tenant profile");
return tenantProfileService.saveTenantProfile(TenantId.SYS_TENANT_ID, tenantProfile);
}
private class TestTbMultipartFile implements TbMultipartFile {
private final MockMultipartFile file;
private TestTbMultipartFile(MockMultipartFile file) {
this.file = file;
}
@Override
public Optional<InputStream> getInputStream() {
try {
return Optional.of(file.getInputStream());
} catch (IOException e) {
return Optional.empty();
}
}
@Override
public String getFileName() {
return file.getName();
}
@Override
public long getFileSize() {
return file.getSize();
}
@Override
public String getContentType() {
return file.getContentType();
}
}
}

1
dao/src/test/java/org/thingsboard/server/dao/service/sql/OtaPackageServiceSqlTest.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.dao.service.sql;
import org.thingsboard.server.dao.service.BaseOtaPackageServiceTest;
import org.thingsboard.server.dao.service.DaoSqlTest;

Loading…
Cancel
Save