From 8a5a4f0995048ed1d7787bb91bccab135220a7c3 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Wed, 28 Apr 2021 12:26:35 +0300 Subject: [PATCH 1/4] Added ability to autogenerate checksum for firmware --- .../server/controller/FirmwareController.java | 13 +++-- .../server/dao/firmware/FirmwareService.java | 4 ++ .../transport/mqtt/MqttTransportHandler.java | 3 -- .../dao/firmware/BaseFirmwareService.java | 48 +++++++++++-------- ui-ngx/src/app/core/http/firmware.service.ts | 10 ++-- .../pages/firmware/firmwares.component.html | 7 +-- .../pages/firmware/firmwares.component.ts | 23 +-------- .../src/app/shared/models/firmware.models.ts | 4 +- 8 files changed, 46 insertions(+), 66 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java b/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java index c8f10b3d78..9fcf5a7ec9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java +++ b/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java @@ -115,7 +115,7 @@ public class FirmwareController extends BaseController { @ResponseBody public Firmware saveFirmwareData(@PathVariable(FIRMWARE_ID) String strFirmwareId, @RequestParam(required = false) String checksum, - @RequestParam(required = false) String checksumAlgorithm, + @RequestParam() String checksumAlgorithm, @RequestBody MultipartFile file) throws ThingsboardException { checkParameter(FIRMWARE_ID, strFirmwareId); try { @@ -129,18 +129,17 @@ public class FirmwareController extends BaseController { firmware.setVersion(info.getVersion()); firmware.setAdditionalInfo(info.getAdditionalInfo()); - byte[] data = file.getBytes(); - if (StringUtils.isEmpty(checksumAlgorithm)) { - checksumAlgorithm = "sha256"; - checksum = Hashing.sha256().hashBytes(data).toString(); + ByteBuffer data = ByteBuffer.wrap(file.getBytes()); + if (StringUtils.isEmpty(checksum)) { + checksum = firmwareService.generateChecksum(checksumAlgorithm, data); } firmware.setChecksumAlgorithm(checksumAlgorithm); firmware.setChecksum(checksum); firmware.setFileName(file.getOriginalFilename()); firmware.setContentType(file.getContentType()); - firmware.setData(ByteBuffer.wrap(data)); - firmware.setDataSize((long) data.length); + firmware.setData(data); + firmware.setDataSize((long) data.array().length); return firmwareService.saveFirmware(firmware); } catch (Exception e) { throw handleException(e); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java index 0fd26fee6e..0ba34ae90f 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java @@ -22,12 +22,16 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; +import java.nio.ByteBuffer; + public interface FirmwareService { FirmwareInfo saveFirmwareInfo(FirmwareInfo firmwareInfo); Firmware saveFirmware(Firmware firmware); + String generateChecksum(String checksumAlgorithm, ByteBuffer data); + Firmware findFirmwareById(TenantId tenantId, FirmwareId firmwareId); FirmwareInfo findFirmwareInfoById(TenantId tenantId, FirmwareId firmwareId); diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 73ca2cdf9d..2b990d6b6b 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -456,9 +456,6 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement deviceSessionCtx.getPayloadAdaptor() .convertToPublish(deviceSessionCtx, firmwareChunk, requestId, chunk) .ifPresent(deviceSessionCtx.getChannel()::writeAndFlush); - if (firmwareChunk != null && chunkSize != firmwareChunk.length) { - scheduler.schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS); - } } catch (Exception e) { log.trace("[{}] Failed to send firmware response!", sessionId, e); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java index 5a80fc75f6..c2b5394ae2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java @@ -101,6 +101,32 @@ public class BaseFirmwareService implements FirmwareService { } } + @Override + public String generateChecksum(String checksumAlgorithm, ByteBuffer data) { + + if (data == null || !data.hasArray() || data.array().length == 0) { + throw new DataValidationException("Firmware data should be specified!"); + } + + HashFunction hashFunction; + switch (checksumAlgorithm) { + case "sha256": + hashFunction = Hashing.sha256(); + break; + case "md5": + hashFunction = Hashing.md5(); + break; + case "crc32": + hashFunction = Hashing.crc32(); + break; + default: + throw new DataValidationException("Unknown checksum algorithm!"); + } + + return hashFunction.hashBytes(data.array()).toString(); + } + + @Override public Firmware findFirmwareById(TenantId tenantId, FirmwareId firmwareId) { log.trace("Executing findFirmwareById [{}]", firmwareId); @@ -217,11 +243,6 @@ public class BaseFirmwareService implements FirmwareService { throw new DataValidationException("Firmware content type should be specified!"); } - ByteBuffer data = firmware.getData(); - if (data == null || !data.hasArray() || data.array().length == 0) { - throw new DataValidationException("Firmware data should be specified!"); - } - if (StringUtils.isEmpty(firmware.getChecksumAlgorithm())) { throw new DataValidationException("Firmware checksum algorithm should be specified!"); } @@ -229,22 +250,7 @@ public class BaseFirmwareService implements FirmwareService { throw new DataValidationException("Firmware checksum should be specified!"); } - HashFunction hashFunction; - switch (firmware.getChecksumAlgorithm()) { - case "sha256": - hashFunction = Hashing.sha256(); - break; - case "md5": - hashFunction = Hashing.md5(); - break; - case "crc32": - hashFunction = Hashing.crc32(); - break; - default: - throw new DataValidationException("Unknown checksum algorithm!"); - } - - String currentChecksum = hashFunction.hashBytes(data.array()).toString(); + String currentChecksum = generateChecksum(firmware.getChecksumAlgorithm(), firmware.getData()); if (!currentChecksum.equals(firmware.getChecksum())) { throw new DataValidationException("Wrong firmware file!"); diff --git a/ui-ngx/src/app/core/http/firmware.service.ts b/ui-ngx/src/app/core/http/firmware.service.ts index 7bf42fb49e..60d16d6bdf 100644 --- a/ui-ngx/src/app/core/http/firmware.service.ts +++ b/ui-ngx/src/app/core/http/firmware.service.ts @@ -20,7 +20,7 @@ import { PageLink } from '@shared/models/page/page-link'; import { defaultHttpOptionsFromConfig, defaultHttpUploadOptions, RequestConfig } from '@core/http/http-utils'; import { Observable } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; -import { Firmware, FirmwareInfo } from '@shared/models/firmware.models'; +import { ChecksumAlgorithm, Firmware, FirmwareInfo } from '@shared/models/firmware.models'; import { catchError, map, mergeMap } from 'rxjs/operators'; import { deepClone, isDefinedAndNotNull } from '@core/utils'; @@ -100,16 +100,16 @@ export class FirmwareService { return this.http.post('/api/firmware', firmware, defaultHttpOptionsFromConfig(config)); } - public uploadFirmwareFile(firmwareId: string, file: File, checksumAlgorithm?: string, + public uploadFirmwareFile(firmwareId: string, file: File, checksumAlgorithm: ChecksumAlgorithm, checksum?: string, config?: RequestConfig): Observable { if (!config) { config = {}; } const formData = new FormData(); formData.append('file', file); - let url = `/api/firmware/${firmwareId}`; - if (checksumAlgorithm && checksum) { - url += `?checksumAlgorithm=${checksumAlgorithm}&checksum=${checksum}`; + let url = `/api/firmware/${firmwareId}?checksumAlgorithm=${checksumAlgorithm}`; + if (checksum) { + url += `&checksum=${checksum}`; } return this.http.post(url, formData, defaultHttpUploadOptions(config.ignoreLoading, config.ignoreErrors, config.resendRequest)); diff --git a/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.html b/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.html index 2911802a75..21a5ad327e 100644 --- a/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.html +++ b/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.html @@ -72,7 +72,6 @@ firmware.checksum-algorithm - {{ checksumAlgorithmTranslationMap.get(checksumAlgorithm) }} @@ -80,11 +79,7 @@ firmware.checksum - - - {{ 'firmware.checksum-required' | translate }} - +
diff --git a/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.ts b/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.ts index 8ffef239b4..5e3c26dfee 100644 --- a/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.ts +++ b/ui-ngx/src/app/modules/home/pages/firmware/firmwares.component.ts @@ -23,7 +23,6 @@ import { EntityTableConfig } from '@home/models/entity/entities-table-config.mod import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { EntityComponent } from '@home/components/entity/entity.component'; import { ChecksumAlgorithm, ChecksumAlgorithmTranslationMap, Firmware } from '@shared/models/firmware.models'; -import { distinctUntilChanged, map, takeUntil } from 'rxjs/operators'; import { ActionNotificationShow } from '@core/notification/notification.actions'; @Component({ @@ -45,26 +44,6 @@ export class FirmwaresComponent extends EntityComponent implements OnI super(store, fb, entityValue, entitiesTableConfigValue); } - ngOnInit() { - super.ngOnInit(); - if (this.isAdd) { - this.entityForm.get('checksumAlgorithm').valueChanges.pipe( - map(algorithm => !!algorithm), - distinctUntilChanged(), - takeUntil(this.destroy$) - ).subscribe( - setAlgorithm => { - if (setAlgorithm) { - this.entityForm.get('checksum').setValidators([Validators.maxLength(1020), Validators.required]); - } else { - this.entityForm.get('checksum').clearValidators(); - } - this.entityForm.get('checksum').updateValueAndValidity({emitEvent: false}); - } - ); - } - } - ngOnDestroy() { super.ngOnDestroy(); this.destroy$.next(); @@ -83,7 +62,7 @@ export class FirmwaresComponent extends EntityComponent implements OnI const form = this.fb.group({ title: [entity ? entity.title : '', [Validators.required, Validators.maxLength(255)]], version: [entity ? entity.version : '', [Validators.required, Validators.maxLength(255)]], - checksumAlgorithm: [entity ? entity.checksumAlgorithm : null], + checksumAlgorithm: [entity && entity.checksumAlgorithm ? entity.checksumAlgorithm : ChecksumAlgorithm.SHA256], checksum: [entity ? entity.checksum : '', Validators.maxLength(1020)], additionalInfo: this.fb.group( { diff --git a/ui-ngx/src/app/shared/models/firmware.models.ts b/ui-ngx/src/app/shared/models/firmware.models.ts index 65dd1761bf..147901a0cc 100644 --- a/ui-ngx/src/app/shared/models/firmware.models.ts +++ b/ui-ngx/src/app/shared/models/firmware.models.ts @@ -38,8 +38,8 @@ export interface FirmwareInfo extends BaseData { version?: string; hasData?: boolean; fileName: string; - checksum?: ChecksumAlgorithm; - checksumAlgorithm?: string; + checksum?: string; + checksumAlgorithm: ChecksumAlgorithm; contentType: string; dataSize?: number; additionalInfo?: any; From 7a4d302c858a98c5958c0ef516cbaadcc24c6c87 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Thu, 29 Apr 2021 09:35:02 +0300 Subject: [PATCH 2/4] Changes according to comments from @ashvayka --- .../server/controller/FirmwareController.java | 10 ++-- .../firmware/DefaultFirmwareStateService.java | 2 +- .../server/dao/firmware/FirmwareService.java | 6 ++- .../server/common/data/FirmwareInfo.java | 3 +- .../data/firmware/ChecksumAlgorithm.java | 26 +++++++++ .../dao/firmware/BaseFirmwareService.java | 54 +++++++++++++++---- .../server/dao/model/sql/FirmwareEntity.java | 6 ++- .../dao/model/sql/FirmwareInfoEntity.java | 8 ++- .../service/BaseDeviceProfileServiceTest.java | 3 +- .../dao/service/BaseDeviceServiceTest.java | 3 +- .../dao/service/BaseFirmwareServiceTest.java | 3 +- .../src/app/shared/models/firmware.models.ts | 16 ++++-- 12 files changed, 115 insertions(+), 25 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/firmware/ChecksumAlgorithm.java diff --git a/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java b/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java index 9fcf5a7ec9..d740d452f3 100644 --- a/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java +++ b/application/src/main/java/org/thingsboard/server/controller/FirmwareController.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.controller; -import com.google.common.hash.Hashing; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.core.io.ByteArrayResource; @@ -33,6 +32,7 @@ import org.springframework.web.multipart.MultipartFile; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.FirmwareInfo; import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -49,6 +49,7 @@ import java.nio.ByteBuffer; public class FirmwareController extends BaseController { public static final String FIRMWARE_ID = "firmwareId"; + public static final String CHECKSUM_ALGORITHM = "checksumAlgorithm"; @PreAuthorize("hasAnyAuthority( 'TENANT_ADMIN')") @RequestMapping(value = "/firmware/{firmwareId}/download", method = RequestMethod.GET) @@ -115,9 +116,10 @@ public class FirmwareController extends BaseController { @ResponseBody public Firmware saveFirmwareData(@PathVariable(FIRMWARE_ID) String strFirmwareId, @RequestParam(required = false) String checksum, - @RequestParam() String checksumAlgorithm, + @RequestParam(CHECKSUM_ALGORITHM) String checksumAlgorithmStr, @RequestBody MultipartFile file) throws ThingsboardException { checkParameter(FIRMWARE_ID, strFirmwareId); + checkParameter(CHECKSUM_ALGORITHM, checksumAlgorithmStr); try { FirmwareId firmwareId = new FirmwareId(toUUID(strFirmwareId)); FirmwareInfo info = checkFirmwareInfoId(firmwareId, Operation.READ); @@ -129,6 +131,8 @@ public class FirmwareController extends BaseController { firmware.setVersion(info.getVersion()); firmware.setAdditionalInfo(info.getAdditionalInfo()); + ChecksumAlgorithm checksumAlgorithm = ChecksumAlgorithm.valueOf(checksumAlgorithmStr.toUpperCase()); + ByteBuffer data = ByteBuffer.wrap(file.getBytes()); if (StringUtils.isEmpty(checksum)) { checksum = firmwareService.generateChecksum(checksumAlgorithm, data); @@ -139,7 +143,7 @@ public class FirmwareController extends BaseController { firmware.setFileName(file.getOriginalFilename()); firmware.setContentType(file.getContentType()); firmware.setData(data); - firmware.setDataSize((long) data.array().length); + firmware.setDataSize((long) data.capacity()); return firmwareService.saveFirmware(firmware); } catch (Exception e) { throw handleException(e); diff --git a/application/src/main/java/org/thingsboard/server/service/firmware/DefaultFirmwareStateService.java b/application/src/main/java/org/thingsboard/server/service/firmware/DefaultFirmwareStateService.java index 6e0e3fd246..58b44a52ce 100644 --- a/application/src/main/java/org/thingsboard/server/service/firmware/DefaultFirmwareStateService.java +++ b/application/src/main/java/org/thingsboard/server/service/firmware/DefaultFirmwareStateService.java @@ -227,7 +227,7 @@ public class DefaultFirmwareStateService implements FirmwareStateService { attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_VERSION, firmware.getVersion()))); attributes.add(new BaseAttributeKvEntry(ts, new LongDataEntry(FIRMWARE_SIZE, firmware.getDataSize()))); - attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_CHECKSUM_ALGORITHM, firmware.getChecksumAlgorithm()))); + attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_CHECKSUM_ALGORITHM, firmware.getChecksumAlgorithm().name()))); attributes.add(new BaseAttributeKvEntry(ts, new StringDataEntry(DataConstants.FIRMWARE_CHECKSUM, firmware.getChecksum()))); telemetryService.saveAndNotify(tenantId, deviceId, DataConstants.SHARED_SCOPE, attributes, new FutureCallback<>() { @Override diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java index 0ba34ae90f..3edc950300 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java @@ -17,6 +17,8 @@ package org.thingsboard.server.dao.firmware; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.FirmwareInfo; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -30,7 +32,9 @@ public interface FirmwareService { Firmware saveFirmware(Firmware firmware); - String generateChecksum(String checksumAlgorithm, ByteBuffer data); + PageData getSupportedChecksumAlgorithms(PageLink pageLink); + + String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data) throws ThingsboardException; Firmware findFirmwareById(TenantId tenantId, FirmwareId firmwareId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/FirmwareInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/FirmwareInfo.java index 3a26763c4f..b0617f16bd 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/FirmwareInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/FirmwareInfo.java @@ -18,6 +18,7 @@ package org.thingsboard.server.common.data; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.id.TenantId; @@ -34,7 +35,7 @@ public class FirmwareInfo extends SearchTextBasedWithAdditionalInfo private boolean hasData; private String fileName; private String contentType; - private String checksumAlgorithm; + private ChecksumAlgorithm checksumAlgorithm; private String checksum; private Long dataSize; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/firmware/ChecksumAlgorithm.java b/common/data/src/main/java/org/thingsboard/server/common/data/firmware/ChecksumAlgorithm.java new file mode 100644 index 0000000000..3998482e35 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/firmware/ChecksumAlgorithm.java @@ -0,0 +1,26 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.firmware; + +public enum ChecksumAlgorithm { + MD5, + SHA256, + SHA384, + SHA512, + CRC32, + MURMUR3_32, + MURMUR3_128 +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java index c2b5394ae2..e3dc4f15cc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java @@ -26,6 +26,9 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.FirmwareInfo; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -35,8 +38,13 @@ import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.tenant.TenantDao; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Optional; import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE; @@ -102,7 +110,7 @@ public class BaseFirmwareService implements FirmwareService { } @Override - public String generateChecksum(String checksumAlgorithm, ByteBuffer data) { + public String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data) throws ThingsboardException { if (data == null || !data.hasArray() || data.array().length == 0) { throw new DataValidationException("Firmware data should be specified!"); @@ -110,22 +118,43 @@ public class BaseFirmwareService implements FirmwareService { HashFunction hashFunction; switch (checksumAlgorithm) { - case "sha256": + case MD5: + hashFunction = Hashing.md5(); + break; + case SHA256: hashFunction = Hashing.sha256(); break; - case "md5": - hashFunction = Hashing.md5(); + case SHA384: + hashFunction = Hashing.sha384(); break; - case "crc32": + case SHA512: + hashFunction = Hashing.sha512(); + break; + case CRC32: hashFunction = Hashing.crc32(); break; + case MURMUR3_32: + hashFunction = Hashing.murmur3_32(); + break; + case MURMUR3_128: + hashFunction = Hashing.murmur3_128(); + break; default: throw new DataValidationException("Unknown checksum algorithm!"); } - return hashFunction.hashBytes(data.array()).toString(); } + @Override + public PageData getSupportedChecksumAlgorithms(PageLink pageLink) { + List methodNames = new ArrayList<>(); + Method[] hashingMethods = Hashing.class.getMethods(); + Arrays.stream(hashingMethods).forEach(method -> { + methodNames.add(method.getName()); + }); + return new PageData<>(methodNames, 1, methodNames.size(), false); + } + @Override public Firmware findFirmwareById(TenantId tenantId, FirmwareId firmwareId) { @@ -150,7 +179,8 @@ public class BaseFirmwareService implements FirmwareService { } @Override - public PageData findTenantFirmwaresByTenantIdAndHasData(TenantId tenantId, boolean hasData, PageLink pageLink) { + public PageData findTenantFirmwaresByTenantIdAndHasData(TenantId tenantId, + boolean hasData, PageLink pageLink) { log.trace("Executing findTenantFirmwaresByTenantIdAndHasData, tenantId [{}], hasData [{}] pageLink [{}]", tenantId, hasData, pageLink); validateId(tenantId, INCORRECT_TENANT_ID + tenantId); validatePageLink(pageLink); @@ -243,14 +273,20 @@ public class BaseFirmwareService implements FirmwareService { throw new DataValidationException("Firmware content type should be specified!"); } - if (StringUtils.isEmpty(firmware.getChecksumAlgorithm())) { + if (firmware.getChecksumAlgorithm() == null) { throw new DataValidationException("Firmware checksum algorithm should be specified!"); } if (StringUtils.isEmpty(firmware.getChecksum())) { throw new DataValidationException("Firmware checksum should be specified!"); } - String currentChecksum = generateChecksum(firmware.getChecksumAlgorithm(), firmware.getData()); + String currentChecksum; + + try { + currentChecksum = generateChecksum(firmware.getChecksumAlgorithm(), firmware.getData()); + } catch (ThingsboardException e) { + throw new DataValidationException(e.getMessage()); + } if (!currentChecksum.equals(firmware.getChecksum())) { throw new DataValidationException("Wrong firmware file!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareEntity.java index ccfa0229bd..8713a33842 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareEntity.java @@ -21,6 +21,7 @@ import lombok.EqualsAndHashCode; import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.server.common.data.Firmware; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseSqlEntity; @@ -30,6 +31,8 @@ import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.Table; import java.nio.ByteBuffer; import java.util.UUID; @@ -68,8 +71,9 @@ public class FirmwareEntity extends BaseSqlEntity implements SearchTex @Column(name = FIRMWARE_CONTENT_TYPE_COLUMN) private String contentType; + @Enumerated(EnumType.STRING) @Column(name = FIRMWARE_CHECKSUM_ALGORITHM_COLUMN) - private String checksumAlgorithm; + private ChecksumAlgorithm checksumAlgorithm; @Column(name = FIRMWARE_CHECKSUM_COLUMN) private String checksum; diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareInfoEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareInfoEntity.java index 3549291e4b..91cfd1b5e1 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareInfoEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/FirmwareInfoEntity.java @@ -22,6 +22,7 @@ import org.hibernate.annotations.Type; import org.hibernate.annotations.TypeDef; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.FirmwareInfo; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.dao.model.BaseSqlEntity; @@ -31,6 +32,8 @@ import org.thingsboard.server.dao.util.mapping.JsonStringType; import javax.persistence.Column; import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; import javax.persistence.Table; import javax.persistence.Transient; import java.util.UUID; @@ -70,8 +73,9 @@ public class FirmwareInfoEntity extends BaseSqlEntity implements S @Column(name = FIRMWARE_CONTENT_TYPE_COLUMN) private String contentType; + @Enumerated(EnumType.STRING) @Column(name = FIRMWARE_CHECKSUM_ALGORITHM_COLUMN) - private String checksumAlgorithm; + private ChecksumAlgorithm checksumAlgorithm; @Column(name = FIRMWARE_CHECKSUM_COLUMN) private String checksum; @@ -108,7 +112,7 @@ public class FirmwareInfoEntity extends BaseSqlEntity implements S } public FirmwareInfoEntity(UUID id, long createdTime, UUID tenantId, String title, String version, - String fileName, String contentType, String checksumAlgorithm, String checksum, Long dataSize, + String fileName, String contentType, ChecksumAlgorithm checksumAlgorithm, String checksum, Long dataSize, Object additionalInfo, boolean hasData) { this.id = id; this.createdTime = createdTime; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java index e377baa2cb..fd1097d7e1 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceProfileServiceTest.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.DeviceProfileInfo; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -101,7 +102,7 @@ public class BaseDeviceProfileServiceTest extends AbstractServiceTest { firmware.setVersion("v1.0"); firmware.setFileName("test.txt"); firmware.setContentType("text/plain"); - firmware.setChecksumAlgorithm("sha256"); + firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); firmware.setData(ByteBuffer.wrap(new byte[]{1})); Firmware savedFirmware = firmwareService.saveFirmware(firmware); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java index 6d7b695ef1..c08b2748d2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseDeviceServiceTest.java @@ -28,6 +28,7 @@ import org.thingsboard.server.common.data.EntitySubtype; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; @@ -187,7 +188,7 @@ public abstract class BaseDeviceServiceTest extends AbstractServiceTest { firmware.setVersion("v1.0"); firmware.setFileName("test.txt"); firmware.setContentType("text/plain"); - firmware.setChecksumAlgorithm("sha256"); + firmware.setChecksumAlgorithm(ChecksumAlgorithm.SHA256); firmware.setChecksum("4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"); firmware.setData(ByteBuffer.wrap(new byte[]{1})); Firmware savedFirmware = firmwareService.saveFirmware(firmware); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseFirmwareServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseFirmwareServiceTest.java index bf96d09922..da459efc9b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseFirmwareServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseFirmwareServiceTest.java @@ -26,6 +26,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.FirmwareInfo; import org.thingsboard.server.common.data.Tenant; +import org.thingsboard.server.common.data.firmware.ChecksumAlgorithm; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -42,7 +43,7 @@ public abstract class BaseFirmwareServiceTest extends AbstractServiceTest { private static final String FILE_NAME = "filename.txt"; private static final String VERSION = "v1.0"; private static final String CONTENT_TYPE = "text/plain"; - private static final String CHECKSUM_ALGORITHM = "sha256"; + private static final ChecksumAlgorithm CHECKSUM_ALGORITHM = ChecksumAlgorithm.SHA256; private static final String CHECKSUM = "4bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459a"; private static final ByteBuffer DATA = ByteBuffer.wrap(new byte[]{1}); diff --git a/ui-ngx/src/app/shared/models/firmware.models.ts b/ui-ngx/src/app/shared/models/firmware.models.ts index 147901a0cc..39775395d7 100644 --- a/ui-ngx/src/app/shared/models/firmware.models.ts +++ b/ui-ngx/src/app/shared/models/firmware.models.ts @@ -19,16 +19,24 @@ import { TenantId } from '@shared/models/id/tenant-id'; import { FirmwareId } from '@shared/models/id/firmware-id'; export enum ChecksumAlgorithm { - MD5 = 'md5', - SHA256 = 'sha256', - CRC32 = 'crc32' + MD5 = 'MD5', + SHA256 = 'SHA256', + SHA384 = 'SHA384', + SHA512 = 'SHA512', + CRC32 = 'CRC32', + MURMUR3_32 = 'MURMUR3_32', + MURMUR3_128 = 'MURMUR3_128' } export const ChecksumAlgorithmTranslationMap = new Map( [ [ChecksumAlgorithm.MD5, 'MD5'], [ChecksumAlgorithm.SHA256, 'SHA-256'], - [ChecksumAlgorithm.CRC32, 'CRC-32'] + [ChecksumAlgorithm.SHA384, 'SHA-384'], + [ChecksumAlgorithm.SHA512, 'SHA-512'], + [ChecksumAlgorithm.CRC32, 'CRC-32'], + [ChecksumAlgorithm.MURMUR3_32, 'MURMUR3-32'], + [ChecksumAlgorithm.MURMUR3_128, 'MURMUR3-128'] ] ); From b45d514b8332a893a076edb455d76c00d65dcc99 Mon Sep 17 00:00:00 2001 From: zbeacon Date: Fri, 30 Apr 2021 08:53:00 +0300 Subject: [PATCH 3/4] Removed unused method for getting supported checksum algorithms --- .../server/dao/firmware/FirmwareService.java | 2 -- .../server/dao/firmware/BaseFirmwareService.java | 11 ----------- 2 files changed, 13 deletions(-) diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java index 2362197a66..6dbd534228 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/firmware/FirmwareService.java @@ -33,8 +33,6 @@ public interface FirmwareService { Firmware saveFirmware(Firmware firmware); - PageData getSupportedChecksumAlgorithms(PageLink pageLink); - String generateChecksum(ChecksumAlgorithm checksumAlgorithm, ByteBuffer data) throws ThingsboardException; Firmware findFirmwareById(TenantId tenantId, FirmwareId firmwareId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java index 4e1913acfe..b17c01ad47 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java @@ -148,17 +148,6 @@ public class BaseFirmwareService implements FirmwareService { return hashFunction.hashBytes(data.array()).toString(); } - @Override - public PageData getSupportedChecksumAlgorithms(PageLink pageLink) { - List methodNames = new ArrayList<>(); - Method[] hashingMethods = Hashing.class.getMethods(); - Arrays.stream(hashingMethods).forEach(method -> { - methodNames.add(method.getName()); - }); - return new PageData<>(methodNames, 1, methodNames.size(), false); - } - - @Override public Firmware findFirmwareById(TenantId tenantId, FirmwareId firmwareId) { log.trace("Executing findFirmwareById [{}]", firmwareId); From a4eb2df8eee8aecccbccfbb27f7e27f19d8fa0fc Mon Sep 17 00:00:00 2001 From: zbeacon Date: Thu, 6 May 2021 13:46:04 +0300 Subject: [PATCH 4/4] Fix for device software/firmware response topic --- .../server/common/data/device/profile/MqttTopics.java | 3 ++- .../server/transport/mqtt/adaptors/JsonMqttAdaptor.java | 4 ++-- .../server/transport/mqtt/adaptors/ProtoMqttAdaptor.java | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java index 6373d41803..2b1aba3c0f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java @@ -82,7 +82,8 @@ public class MqttTopics { public static final String DEVICE_FIRMWARE_REQUEST_TOPIC_PATTERN = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + REQUEST + "/" + REQUEST_ID_PATTERN + CHUNK + CHUNK_PATTERN; public static final String DEVICE_FIRMWARE_RESPONSES_TOPIC = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + RESPONSE + "/" + SUB_TOPIC + CHUNK + SUB_TOPIC; public static final String DEVICE_FIRMWARE_ERROR_TOPIC = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + ERROR; - public static final String DEVICE_FIRMWARE_RESPONSES_TOPIC_FORMAT = BASE_DEVICE_API_TOPIC_V2 + "%s" + RESPONSE + "/"+ "%s" + CHUNK + "%d"; + + public static final String DEVICE_SOFTWARE_FIRMWARE_RESPONSES_TOPIC_FORMAT = BASE_DEVICE_API_TOPIC_V2 + "/%s" + RESPONSE + "/%s" + CHUNK + "%d"; public static final String DEVICE_SOFTWARE_REQUEST_TOPIC_PATTERN = BASE_DEVICE_API_TOPIC_V2 + SOFTWARE + REQUEST + "/" + REQUEST_ID_PATTERN + CHUNK + CHUNK_PATTERN; public static final String DEVICE_SOFTWARE_RESPONSES_TOPIC = BASE_DEVICE_API_TOPIC_V2 + SOFTWARE + RESPONSE + "/" + SUB_TOPIC + CHUNK + SUB_TOPIC; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index dbda48d15d..a9d7b3e2ea 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -44,7 +44,7 @@ import java.util.Optional; import java.util.Set; import java.util.UUID; -import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC_FORMAT; +import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_SOFTWARE_FIRMWARE_RESPONSES_TOPIC_FORMAT; /** @@ -156,7 +156,7 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { @Override public Optional convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk, FirmwareType firmwareType) { - return Optional.of(createMqttPublishMsg(ctx, String.format(DEVICE_FIRMWARE_RESPONSES_TOPIC_FORMAT, firmwareType.getKeyPrefix(), requestId, chunk), firmwareChunk)); + return Optional.of(createMqttPublishMsg(ctx, String.format(DEVICE_SOFTWARE_FIRMWARE_RESPONSES_TOPIC_FORMAT, firmwareType.getKeyPrefix(), requestId, chunk), firmwareChunk)); } public static JsonElement validateJsonPayload(UUID sessionId, ByteBuf payloadData) throws AdaptorException { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java index 29df08e9c3..08a2f9abe3 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java @@ -39,7 +39,7 @@ import org.thingsboard.server.transport.mqtt.session.MqttDeviceAwareSessionConte import java.util.Optional; -import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC_FORMAT; +import static org.thingsboard.server.common.data.device.profile.MqttTopics.DEVICE_SOFTWARE_FIRMWARE_RESPONSES_TOPIC_FORMAT; @Component @Slf4j @@ -169,7 +169,7 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor { @Override public Optional convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk, FirmwareType firmwareType) throws AdaptorException { - return Optional.of(createMqttPublishMsg(ctx, String.format(DEVICE_FIRMWARE_RESPONSES_TOPIC_FORMAT, firmwareType.getKeyPrefix(), requestId, chunk), firmwareChunk)); + return Optional.of(createMqttPublishMsg(ctx, String.format(DEVICE_SOFTWARE_FIRMWARE_RESPONSES_TOPIC_FORMAT, firmwareType.getKeyPrefix(), requestId, chunk), firmwareChunk)); } @Override