Browse Source

implemented firmware update for the http and mqtt transports

pull/4380/head
YevhenBondarenko 6 years ago
parent
commit
f55f0ecb94
  1. 65
      application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java
  2. 5
      application/src/main/resources/thingsboard.yml
  3. 33
      common/cache/src/main/java/org/thingsboard/server/cache/firmware/AbstractRedisFirmwareCache.java
  4. 60
      common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheReader.java
  5. 38
      common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheWriter.java
  6. 22
      common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheReader.java
  7. 20
      common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheWriter.java
  8. 49
      common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheReader.java
  9. 38
      common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheWriter.java
  10. 1
      common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java
  11. 10
      common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java
  12. 22
      common/queue/src/main/proto/queue.proto
  13. 2
      common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java
  14. 65
      common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java
  15. 1
      common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisSecurityStore.java
  16. 111
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java
  17. 10
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java
  18. 18
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java
  19. 22
      common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java
  20. 8
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java
  21. 4
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java
  22. 12
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java
  23. 18
      common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java
  24. 19
      dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java

65
application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java

@ -23,16 +23,19 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.protobuf.ByteString;
import lombok.extern.slf4j.Slf4j;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cache.firmware.FirmwareCacheWriter;
import org.thingsboard.server.common.data.ApiUsageState;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.Firmware;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials;
import org.thingsboard.server.common.data.device.credentials.ProvisionDeviceCredentialsData;
@ -40,6 +43,7 @@ import org.thingsboard.server.common.data.device.profile.ProvisionDeviceProfileC
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.DeviceProfileId;
import org.thingsboard.server.common.data.id.FirmwareId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.relation.EntityRelation;
import org.thingsboard.server.common.data.security.DeviceCredentials;
@ -55,6 +59,7 @@ import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.device.provision.ProvisionFailedException;
import org.thingsboard.server.dao.device.provision.ProvisionRequest;
import org.thingsboard.server.dao.device.provision.ProvisionResponse;
import org.thingsboard.server.dao.firmware.FirmwareService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.resource.TbResourceService;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
@ -66,7 +71,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFro
import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetResourceRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg;
@ -109,6 +113,8 @@ public class DefaultTransportApiService implements TransportApiService {
private final DataDecodingEncodingService dataDecodingEncodingService;
private final DeviceProvisionService deviceProvisionService;
private final TbResourceService resourceService;
private final FirmwareService firmwareService;
private final FirmwareCacheWriter firmwareCacheWriter;
private final ConcurrentMap<String, ReentrantLock> deviceCreationLocks = new ConcurrentHashMap<>();
@ -117,7 +123,7 @@ public class DefaultTransportApiService implements TransportApiService {
RelationService relationService, DeviceCredentialsService deviceCredentialsService,
DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService,
TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService,
DeviceProvisionService deviceProvisionService, TbResourceService resourceService) {
DeviceProvisionService deviceProvisionService, TbResourceService resourceService, FirmwareService firmwareService, CacheManager cacheManager, FirmwareCacheWriter firmwareCacheWriter) {
this.deviceProfileCache = deviceProfileCache;
this.tenantProfileCache = tenantProfileCache;
this.apiUsageStateService = apiUsageStateService;
@ -130,6 +136,8 @@ public class DefaultTransportApiService implements TransportApiService {
this.dataDecodingEncodingService = dataDecodingEncodingService;
this.deviceProvisionService = deviceProvisionService;
this.resourceService = resourceService;
this.firmwareService = firmwareService;
this.firmwareCacheWriter = firmwareCacheWriter;
}
@Override
@ -166,6 +174,9 @@ public class DefaultTransportApiService implements TransportApiService {
} else if (transportApiRequestMsg.hasResourceRequestMsg()) {
return Futures.transform(handle(transportApiRequestMsg.getResourceRequestMsg()),
value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
} else if (transportApiRequestMsg.hasFirmwareRequestMsg()) {
return Futures.transform(handle(transportApiRequestMsg.getFirmwareRequestMsg()),
value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
}
return Futures.transform(getEmptyTransportApiResponseFuture(),
value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor());
@ -313,14 +324,14 @@ public class DefaultTransportApiService implements TransportApiService {
} catch (ProvisionFailedException e) {
return Futures.immediateFuture(getTransportApiResponseMsg(
new DeviceCredentials(),
TransportProtos.ProvisionResponseStatus.valueOf(e.getMessage())));
TransportProtos.ResponseStatus.valueOf(e.getMessage())));
}
return Futures.transform(provisionResponseFuture, provisionResponse -> getTransportApiResponseMsg(provisionResponse.getDeviceCredentials(), TransportProtos.ProvisionResponseStatus.SUCCESS),
return Futures.transform(provisionResponseFuture, provisionResponse -> getTransportApiResponseMsg(provisionResponse.getDeviceCredentials(), TransportProtos.ResponseStatus.SUCCESS),
dbCallbackExecutorService);
}
private TransportApiResponseMsg getTransportApiResponseMsg(DeviceCredentials deviceCredentials, TransportProtos.ProvisionResponseStatus status) {
if (!status.equals(ProvisionResponseStatus.SUCCESS)) {
private TransportApiResponseMsg getTransportApiResponseMsg(DeviceCredentials deviceCredentials, TransportProtos.ResponseStatus status) {
if (!status.equals(TransportProtos.ResponseStatus.SUCCESS)) {
return TransportApiResponseMsg.newBuilder().setProvisionDeviceResponseMsg(TransportProtos.ProvisionDeviceResponseMsg.newBuilder().setStatus(status).build()).build();
}
TransportProtos.ProvisionDeviceResponseMsg.Builder provisionResponse = TransportProtos.ProvisionDeviceResponseMsg.newBuilder()
@ -438,6 +449,46 @@ public class DefaultTransportApiService implements TransportApiService {
}
}
private ListenableFuture<TransportApiResponseMsg> handle(TransportProtos.GetFirmwareRequestMsg requestMsg) {
TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB()));
DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB()));
Device device = deviceService.findDeviceById(tenantId, deviceId);
if (device == null) {
return getEmptyTransportApiResponseFuture();
}
FirmwareId firmwareId = device.getFirmwareId();
if (firmwareId == null) {
firmwareId = deviceProfileCache.find(device.getDeviceProfileId()).getFirmwareId();
}
TransportProtos.GetFirmwareResponseMsg.Builder builder = TransportProtos.GetFirmwareResponseMsg.newBuilder();
if (firmwareId == null) {
builder.setResponseStatus(TransportProtos.ResponseStatus.NOT_FOUND);
} else {
Firmware firmware = firmwareService.findFirmwareById(tenantId, firmwareId);
if (firmware == null) {
builder.setResponseStatus(TransportProtos.ResponseStatus.NOT_FOUND);
} else {
builder.setResponseStatus(TransportProtos.ResponseStatus.SUCCESS);
builder.setFirmwareIdMSB(firmwareId.getId().getMostSignificantBits());
builder.setFirmwareIdLSB(firmwareId.getId().getLeastSignificantBits());
builder.setFileName(firmware.getFileName());
builder.setContentType(firmware.getContentType());
firmwareCacheWriter.put(firmwareId.toString(), firmware.getData().array());
}
}
return Futures.immediateFuture(
TransportApiResponseMsg.newBuilder()
.setFirmwareResponseMsg(builder.build())
.build());
}
private ListenableFuture<TransportApiResponseMsg> handleRegistration(TransportProtos.LwM2MRegistrationRequestMsg msg) {
TenantId tenantId = new TenantId(UUID.fromString(msg.getTenantId()));
String deviceName = msg.getEndpoint();

5
application/src/main/resources/thingsboard.yml

@ -281,7 +281,7 @@ actors:
tenant:
create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}"
session:
max_concurrent_sessions_per_device: "${ACTORS_MAX_CONCURRENT_SESSION_PER_DEVICE:1}"
max_concurrent_sessions_per_device: "${ACTORS_MAX_CONCURRENT_SESSION_PER_DEVICE:2}"
sync:
# Default timeout for processing request using synchronous session (HTTP, CoAP) in milliseconds
timeout: "${ACTORS_SESSION_SYNC_TIMEOUT:10000}"
@ -365,6 +365,9 @@ caffeine:
tokensOutdatageTime:
timeToLiveInMinutes: 20000
maxSize: 10000
firmwares:
timeToLiveInMinutes: 1440
maxSize: 100
redis:
# standalone or cluster

33
common/cache/src/main/java/org/thingsboard/server/cache/firmware/AbstractRedisFirmwareCache.java

@ -0,0 +1,33 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.firmware;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE;
public abstract class AbstractRedisFirmwareCache {
protected final RedisConnectionFactory redisConnectionFactory;
protected AbstractRedisFirmwareCache(RedisConnectionFactory redisConnectionFactory) {
this.redisConnectionFactory = redisConnectionFactory;
}
protected byte[] toFirmwareCacheKey(String key) {
return String.format("%s::%s", FIRMWARE_CACHE, key).getBytes();
}
}

60
common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheReader.java

@ -0,0 +1,60 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.firmware;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE;
@Service
@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport') && ('${cache.type:null}'=='caffeine' || '${cache.type:null}'=='caffeine')")
public class CaffeineFirmwareCacheReader implements FirmwareCacheReader {
private final CacheManager cacheManager;
public CaffeineFirmwareCacheReader(CacheManager cacheManager) {
this.cacheManager = cacheManager;
}
@Override
public byte[] get(String key) {
return get(key, 0, 0);
}
@Override
public byte[] get(String key, int chunkSize, int chunk) {
byte[] data = cacheManager.getCache(FIRMWARE_CACHE).get(key, byte[].class);
if (chunkSize < 1) {
return data;
}
if (data != null && data.length > 0) {
int startIndex = chunkSize * chunk;
int size = Math.min(data.length - startIndex, chunkSize);
if (startIndex < data.length && size > 0) {
byte[] result = new byte[size];
System.arraycopy(data, startIndex, result, 0, size);
return result;
}
}
return new byte[0];
}
}

38
common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheWriter.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.firmware;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE;
@Service
@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='core') && ('${cache.type:null}'=='caffeine' || '${cache.type:null}'=='caffeine')")
public class CaffeineFirmwareCacheWriter implements FirmwareCacheWriter {
private final CacheManager cacheManager;
public CaffeineFirmwareCacheWriter(CacheManager cacheManager) {
this.cacheManager = cacheManager;
}
@Override
public void put(String key, byte[] value) {
cacheManager.getCache(FIRMWARE_CACHE).putIfAbsent(key, value);
}
}

22
common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheReader.java

@ -0,0 +1,22 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.firmware;
public interface FirmwareCacheReader {
byte[] get(String key);
byte[] get(String key, int chunkSize, int chunk);
}

20
common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheWriter.java

@ -0,0 +1,20 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.firmware;
public interface FirmwareCacheWriter {
void put(String key, byte[] value);
}

49
common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheReader.java

@ -0,0 +1,49 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.firmware;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
@Service
@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport') && '${cache.type:null}'=='redis'")
public class RedisFirmwareCacheReader extends AbstractRedisFirmwareCache implements FirmwareCacheReader {
public RedisFirmwareCacheReader(RedisConnectionFactory redisConnectionFactory) {
super(redisConnectionFactory);
}
@Override
public byte[] get(String key) {
return get(key, 0, 0);
}
@Override
public byte[] get(String key, int chunkSize, int chunk) {
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
if (chunkSize == 0) {
return connection.get(toFirmwareCacheKey(key));
}
int startIndex = chunkSize * chunk;
int endIndex = startIndex + chunkSize - 1;
return connection.getRange(toFirmwareCacheKey(key), startIndex, endIndex);
}
}
}

38
common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheWriter.java

@ -0,0 +1,38 @@
/**
* Copyright © 2016-2021 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.cache.firmware;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.data.redis.connection.RedisConnection;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.stereotype.Service;
@Service
@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='core') && '${cache.type:null}'=='redis'")
public class RedisFirmwareCacheWriter extends AbstractRedisFirmwareCache implements FirmwareCacheWriter {
public RedisFirmwareCacheWriter(RedisConnectionFactory redisConnectionFactory) {
super(redisConnectionFactory);
}
@Override
public void put(String key, byte[] value) {
try (RedisConnection connection = redisConnectionFactory.getConnection()) {
connection.set(toFirmwareCacheKey(key), value);
}
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java

@ -28,4 +28,5 @@ public class CacheConstants {
public static final String DEVICE_PROFILE_CACHE = "deviceProfiles";
public static final String ATTRIBUTES_CACHE = "attributes";
public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime";
public static final String FIRMWARE_CACHE = "firmwares";
}

10
common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java

@ -30,6 +30,9 @@ public class MqttTopics {
private static final String CLAIM = "/claim";
private static final String SUB_TOPIC = "+";
private static final String PROVISION = "/provision";
private static final String FIRMWARE = "/fw";
private static final String CHUNK = "/chunk/";
private static final String ERROR = "/error";
private static final String ATTRIBUTES_RESPONSE = ATTRIBUTES + RESPONSE;
private static final String ATTRIBUTES_REQUEST = ATTRIBUTES + REQUEST;
@ -69,6 +72,13 @@ public class MqttTopics {
public static final String GATEWAY_ATTRIBUTES_REQUEST_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_REQUEST;
public static final String GATEWAY_ATTRIBUTES_RESPONSE_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_RESPONSE;
// v2 topics
public static final String BASE_DEVICE_API_TOPIC_V2 = "v2";
public static final String DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + RESPONSE + "/";
public static final String DEVICE_FIRMWARE_RESPONSES_TOPIC = DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX + SUB_TOPIC + CHUNK + SUB_TOPIC;
public static final String DEVICE_FIRMWARE_ERROR_TOPIC = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + ERROR;
private MqttTopics() {
}
}

22
common/queue/src/main/proto/queue.proto

@ -336,17 +336,33 @@ message ProvisionDeviceCredentialsMsg {
}
message ProvisionDeviceResponseMsg {
ProvisionResponseStatus status = 1;
ResponseStatus status = 1;
CredentialsType credentialsType = 2;
string credentialsValue = 3;
}
enum ProvisionResponseStatus {
enum ResponseStatus {
UNKNOWN = 0;
SUCCESS = 1;
NOT_FOUND = 2;
FAILURE = 3;
}
message GetFirmwareRequestMsg {
int64 deviceIdMSB = 1;
int64 deviceIdLSB = 2;
int64 tenantIdMSB = 3;
int64 tenantIdLSB = 4;
}
message GetFirmwareResponseMsg {
ResponseStatus responseStatus = 1;
int64 firmwareIdMSB = 2;
int64 firmwareIdLSB = 3;
string contentType = 4;
string fileName = 5;
}
//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.
message SubscriptionInfoProto {
int64 lastActivityTime = 1;
@ -552,6 +568,7 @@ message TransportApiRequestMsg {
ProvisionDeviceRequestMsg provisionDeviceRequestMsg = 7;
ValidateDeviceLwM2MCredentialsRequestMsg validateDeviceLwM2MCredentialsRequestMsg = 8;
GetResourceRequestMsg resourceRequestMsg = 9;
GetFirmwareRequestMsg firmwareRequestMsg = 10;
}
/* Response from ThingsBoard Core Service to Transport Service */
@ -562,6 +579,7 @@ message TransportApiResponseMsg {
ProvisionDeviceResponseMsg provisionDeviceResponseMsg = 4;
LwM2MResponseMsg lwM2MResponseMsg = 6;
GetResourceResponseMsg resourceResponseMsg = 7;
GetFirmwareResponseMsg firmwareResponseMsg = 8;
}
/* Messages that are handled by ThingsBoard Core Service */

2
common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java

@ -360,7 +360,7 @@ public class CoapTransportResource extends AbstractCoapTransportResource {
@Override
public void onSuccess(TransportProtos.ProvisionDeviceResponseMsg msg) {
CoAP.ResponseCode responseCode = CoAP.ResponseCode.CREATED;
if (!msg.getStatus().equals(TransportProtos.ProvisionResponseStatus.SUCCESS)) {
if (!msg.getStatus().equals(TransportProtos.ResponseStatus.SUCCESS)) {
responseCode = CoAP.ResponseCode.BAD_REQUEST;
}
if (payloadType.equals(TransportPayloadType.JSON)) {

65
common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java

@ -20,7 +20,10 @@ import com.google.gson.JsonParser;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression;
import org.springframework.core.io.ByteArrayResource;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.PathVariable;
@ -41,7 +44,6 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator;
import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto;
import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg;
@ -53,7 +55,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs
import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
import javax.servlet.http.HttpServletRequest;
@ -204,6 +205,23 @@ public class DeviceApiController {
return responseWriter;
}
@RequestMapping(value = "/{deviceToken}/firmware", method = RequestMethod.GET)
public DeferredResult<ResponseEntity> getFirmware(@PathVariable("deviceToken") String deviceToken,
@RequestParam(value = "chunkSize", required = false, defaultValue = "0") int chunkSize,
@RequestParam(value = "chunk", required = false, defaultValue = "0") int chunk) {
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<>();
transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(),
new DeviceAuthCallback(transportContext, responseWriter, sessionInfo -> {
TransportProtos.GetFirmwareRequestMsg requestMsg = TransportProtos.GetFirmwareRequestMsg.newBuilder()
.setTenantIdMSB(sessionInfo.getTenantIdMSB())
.setTenantIdLSB(sessionInfo.getTenantIdLSB())
.setDeviceIdMSB(sessionInfo.getDeviceIdMSB())
.setDeviceIdLSB(sessionInfo.getDeviceIdLSB()).build();
transportContext.getTransportService().process(sessionInfo, requestMsg, new GetFirmwareCallback(responseWriter, chunkSize, chunk));
}));
return responseWriter;
}
@RequestMapping(value = "/provision", method = RequestMethod.POST)
public DeferredResult<ResponseEntity> provisionDevice(@RequestBody String json, HttpServletRequest httpRequest) {
DeferredResult<ResponseEntity> responseWriter = new DeferredResult<>();
@ -258,6 +276,41 @@ public class DeviceApiController {
}
}
private class GetFirmwareCallback implements TransportServiceCallback<TransportProtos.GetFirmwareResponseMsg> {
private final DeferredResult<ResponseEntity> responseWriter;
private final int chuckSize;
private final int chuck;
GetFirmwareCallback(DeferredResult<ResponseEntity> responseWriter, int chuckSize, int chuck) {
this.responseWriter = responseWriter;
this.chuckSize = chuckSize;
this.chuck = chuck;
}
@Override
public void onSuccess(TransportProtos.GetFirmwareResponseMsg firmwareResponseMsg) {
if (!TransportProtos.ResponseStatus.SUCCESS.equals(firmwareResponseMsg.getResponseStatus())) {
responseWriter.setResult(new ResponseEntity<>(HttpStatus.NOT_FOUND));
} else {
String firmwareId = new UUID(firmwareResponseMsg.getFirmwareIdMSB(), firmwareResponseMsg.getFirmwareIdLSB()).toString();
ByteArrayResource resource = new ByteArrayResource(transportContext.getFirmwareCacheReader().get(firmwareId, chuckSize, chuck));
ResponseEntity<ByteArrayResource> response = ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + firmwareResponseMsg.getFileName())
.header("x-filename", firmwareResponseMsg.getFileName())
.contentLength(resource.contentLength())
.contentType(parseMediaType(firmwareResponseMsg.getContentType()))
.body(resource);
responseWriter.setResult(response);
}
}
@Override
public void onError(Throwable e) {
log.warn("Failed to process request", e);
responseWriter.setResult(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR));
}
}
private static class SessionCloseOnErrorCallback implements TransportServiceCallback<Void> {
private final TransportService transportService;
private final SessionInfoProto sessionInfo;
@ -338,4 +391,12 @@ public class DeviceApiController {
.build(), TransportServiceCallback.EMPTY);
}
private static MediaType parseMediaType(String contentType) {
try {
return MediaType.parseMediaType(contentType);
} catch (Exception e) {
return MediaType.APPLICATION_OCTET_STREAM;
}
}
}

1
common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisSecurityStore.java

@ -29,7 +29,6 @@ import redis.clients.jedis.ScanResult;
import java.util.Collection;
import java.util.LinkedList;
@Service
public class TbLwM2mRedisSecurityStore implements EditableSecurityStore {
private static final String SEC_EP = "SEC#EP#";

111
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java

@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.DeviceTransportType;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.MqttTopics;
import org.thingsboard.server.common.data.id.FirmwareId;
import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.common.msg.tools.TbRateLimitsException;
import org.thingsboard.server.common.transport.SessionMsgListener;
@ -69,10 +70,10 @@ import org.thingsboard.server.transport.mqtt.session.MqttTopicMatcher;
import org.thingsboard.server.transport.mqtt.util.SslUtil;
import javax.net.ssl.SSLPeerUnverifiedException;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.cert.Certificate;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
@ -80,7 +81,10 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.amazonaws.util.StringUtils.UTF8;
import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_ACCEPTED;
import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_NOT_AUTHORIZED;
import static io.netty.handler.codec.mqtt.MqttMessageType.CONNACK;
@ -99,6 +103,10 @@ import static io.netty.handler.codec.mqtt.MqttQoS.FAILURE;
@Slf4j
public class MqttTransportHandler extends ChannelInboundHandlerAdapter implements GenericFutureListener<Future<? super Void>>, SessionMsgListener {
private static final Pattern FW_PATTERN = Pattern.compile("v2/fw/request/(?<requestId>\\d+)/chunk/(?<chunk>\\d+)");
private static final String PAYLOAD_TOO_LARGE = "PAYLOAD_TOO_LARGE";
private static final MqttQoS MAX_SUPPORTED_QOS_LVL = AT_LEAST_ONCE;
private final UUID sessionId;
@ -112,6 +120,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
private volatile InetSocketAddress address;
private volatile GatewaySessionHandler gatewaySessionHandler;
private final ConcurrentHashMap<String, String> fwSessions;
private final ConcurrentHashMap<String, Integer> fwChunkSizes;
MqttTransportHandler(MqttTransportContext context, SslHandler sslHandler) {
this.sessionId = UUID.randomUUID();
this.context = context;
@ -120,6 +131,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
this.sslHandler = sslHandler;
this.mqttQoSMap = new ConcurrentHashMap<>();
this.deviceSessionCtx = new DeviceSessionCtx(sessionId, mqttQoSMap, context);
this.fwSessions = new ConcurrentHashMap<>();
this.fwChunkSizes = new ConcurrentHashMap<>();
}
@Override
@ -280,6 +293,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
private void processDevicePublish(ChannelHandlerContext ctx, MqttPublishMessage mqttMsg, String topicName, int msgId) {
try {
Matcher fwMatcher;
MqttTransportAdaptor payloadAdaptor = deviceSessionCtx.getPayloadAdaptor();
if (deviceSessionCtx.isDeviceAttributesTopic(topicName)) {
TransportProtos.PostAttributeMsg postAttributeMsg = payloadAdaptor.convertToPostAttributes(deviceSessionCtx, mqttMsg);
@ -299,6 +313,38 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
} else if (topicName.equals(MqttTopics.DEVICE_CLAIM_TOPIC)) {
TransportProtos.ClaimDeviceMsg claimDeviceMsg = payloadAdaptor.convertToClaimDevice(deviceSessionCtx, mqttMsg);
transportService.process(deviceSessionCtx.getSessionInfo(), claimDeviceMsg, getPubAckCallback(ctx, msgId, claimDeviceMsg));
} else if ((fwMatcher = FW_PATTERN.matcher(topicName)).find()) {
String payload = mqttMsg.content().toString(UTF8);
int chunkSize = payload != null ? Integer.parseInt(payload) : 0;
String requestId = fwMatcher.group("requestId");
int chunk = Integer.parseInt(fwMatcher.group("chunk"));
if (chunkSize > 0) {
this.fwChunkSizes.put(requestId, chunkSize);
} else {
chunkSize = fwChunkSizes.getOrDefault(requestId, 0);
}
if (chunkSize > context.getMaxPayloadSize()) {
sendFirmwareError(ctx, PAYLOAD_TOO_LARGE);
return;
}
String firmwareId = fwSessions.get(requestId);
if (firmwareId != null) {
sendFirmware(ctx, mqttMsg.variableHeader().packetId(), firmwareId, requestId, chunkSize, chunk);
} else {
TransportProtos.SessionInfoProto sessionInfo = deviceSessionCtx.getSessionInfo();
TransportProtos.GetFirmwareRequestMsg getFirmwareRequestMsg = TransportProtos.GetFirmwareRequestMsg.newBuilder()
.setDeviceIdMSB(sessionInfo.getDeviceIdMSB())
.setDeviceIdLSB(sessionInfo.getDeviceIdLSB())
.setTenantIdMSB(sessionInfo.getTenantIdMSB())
.setTenantIdLSB(sessionInfo.getTenantIdLSB())
.build();
transportService.process(deviceSessionCtx.getSessionInfo(), getFirmwareRequestMsg,
new FirmwareCallback(ctx, mqttMsg.variableHeader().packetId(), getFirmwareRequestMsg, requestId, chunkSize, chunk));
}
} else {
transportService.reportActivity(deviceSessionCtx.getSessionInfo());
ack(ctx, msgId);
@ -366,6 +412,65 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
}
}
private class FirmwareCallback implements TransportServiceCallback<TransportProtos.GetFirmwareResponseMsg> {
private final ChannelHandlerContext ctx;
private final int msgId;
private final TransportProtos.GetFirmwareRequestMsg msg;
private final String requestId;
private final int chunkSize;
private final int chunk;
FirmwareCallback(ChannelHandlerContext ctx, int msgId, TransportProtos.GetFirmwareRequestMsg msg, String requestId, int chunkSize, int chunk) {
this.ctx = ctx;
this.msgId = msgId;
this.msg = msg;
this.requestId = requestId;
this.chunkSize = chunkSize;
this.chunk = chunk;
}
@Override
public void onSuccess(TransportProtos.GetFirmwareResponseMsg response) {
if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus())) {
FirmwareId firmwareId = new FirmwareId(new UUID(response.getFirmwareIdMSB(), response.getFirmwareIdLSB()));
fwSessions.put(requestId, firmwareId.toString());
sendFirmware(ctx, msgId, firmwareId.toString(), requestId, chunkSize, chunk);
} else {
sendFirmwareError(ctx, response.getResponseStatus().toString());
}
}
@Override
public void onError(Throwable e) {
log.trace("[{}] Failed to get firmware: {}", sessionId, msg, e);
processDisconnect(ctx);
}
}
private void sendFirmware(ChannelHandlerContext ctx, int msgId, String firmwareId, String requestId, int chunkSize, int chunk) {
log.trace("[{}] Send firmware [{}] to device!", sessionId, firmwareId);
ack(ctx, msgId);
try {
byte[] firmwareChunk = context.getFirmwareCacheReader().get(firmwareId, chunkSize, chunk);
deviceSessionCtx.getPayloadAdaptor()
.convertToPublish(deviceSessionCtx, firmwareChunk, requestId, chunk)
.ifPresent(deviceSessionCtx.getChannel()::writeAndFlush);
if (firmwareChunk != null && chunkSize != firmwareChunk.length) {
scheduler.schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS);
}
} catch (Exception e) {
log.trace("[{}] Failed to send firmware response!", sessionId, e);
}
}
private void sendFirmwareError(ChannelHandlerContext ctx, String error) {
log.warn("[{}] {}", sessionId, error);
deviceSessionCtx.getChannel().writeAndFlush(deviceSessionCtx
.getPayloadAdaptor()
.createMqttPublishMsg(deviceSessionCtx, MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC, error.getBytes()));
processDisconnect(ctx);
}
private void processSubscribe(ChannelHandlerContext ctx, MqttSubscribeMessage mqttMsg) {
if (!checkConnected(ctx, mqttMsg)) {
return;
@ -396,6 +501,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement
case MqttTopics.GATEWAY_RPC_TOPIC:
case MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC:
case MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC:
case MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC:
case MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC:
registerSubQoS(topic, grantedQoSList, reqQoS);
break;
default:

10
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java

@ -21,8 +21,6 @@ import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.gson.JsonSyntaxException;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.handler.codec.mqtt.MqttFixedHeader;
import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.handler.codec.mqtt.MqttMessageType;
@ -31,10 +29,10 @@ import io.netty.handler.codec.mqtt.MqttPublishVariableHeader;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import org.thingsboard.server.common.data.device.profile.MqttTopics;
import org.thingsboard.server.common.transport.adaptor.AdaptorException;
import org.thingsboard.server.common.transport.adaptor.JsonConverter;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.common.data.device.profile.MqttTopics;
import org.thingsboard.server.transport.mqtt.session.MqttDeviceAwareSessionContext;
import java.nio.charset.Charset;
@ -55,7 +53,6 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor {
protected static final Charset UTF8 = StandardCharsets.UTF_8;
private static final Gson GSON = new Gson();
private static final ByteBufAllocator ALLOCATOR = new UnpooledByteBufAllocator(false);
@Override
public TransportProtos.PostTelemetryMsg convertToPostTelemetry(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException {
@ -153,6 +150,11 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor {
return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC, JsonConverter.toJson(provisionResponse)));
}
@Override
public Optional<MqttMessage> convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk) {
return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX + requestId + "/" + chunk, firmwareChunk));
}
public static JsonElement validateJsonPayload(UUID sessionId, ByteBuf payloadData) throws AdaptorException {
String payload = validatePayload(sessionId, payloadData, false);
try {

18
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java

@ -15,8 +15,14 @@
*/
package org.thingsboard.server.transport.mqtt.adaptors;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.handler.codec.mqtt.MqttFixedHeader;
import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.handler.codec.mqtt.MqttMessageType;
import io.netty.handler.codec.mqtt.MqttPublishMessage;
import io.netty.handler.codec.mqtt.MqttPublishVariableHeader;
import org.thingsboard.server.common.transport.adaptor.AdaptorException;
import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg;
@ -39,6 +45,8 @@ import java.util.Optional;
*/
public interface MqttTransportAdaptor {
ByteBufAllocator ALLOCATOR = new UnpooledByteBufAllocator(false);
PostTelemetryMsg convertToPostTelemetry(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException;
PostAttributeMsg convertToPostAttributes(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException;
@ -69,4 +77,14 @@ public interface MqttTransportAdaptor {
Optional<MqttMessage> convertToPublish(MqttDeviceAwareSessionContext ctx, ProvisionDeviceResponseMsg provisionResponse) throws AdaptorException;
Optional<MqttMessage> convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk) throws AdaptorException;
default MqttPublishMessage createMqttPublishMsg(MqttDeviceAwareSessionContext ctx, String topic, byte[] payloadInBytes) {
MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(MqttMessageType.PUBLISH, false, ctx.getQoSForTopic(topic), false, 0);
MqttPublishVariableHeader header = new MqttPublishVariableHeader(topic, ctx.nextMsgId());
ByteBuf payload = ALLOCATOR.buffer();
payload.writeBytes(payloadInBytes);
return new MqttPublishMessage(mqttFixedHeader, header, payload);
}
}

22
common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java

@ -21,13 +21,8 @@ import com.google.protobuf.DynamicMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.google.protobuf.util.JsonFormat;
import io.netty.buffer.ByteBuf;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.handler.codec.mqtt.MqttFixedHeader;
import io.netty.handler.codec.mqtt.MqttMessage;
import io.netty.handler.codec.mqtt.MqttMessageType;
import io.netty.handler.codec.mqtt.MqttPublishMessage;
import io.netty.handler.codec.mqtt.MqttPublishVariableHeader;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
@ -46,8 +41,6 @@ import java.util.Optional;
@Slf4j
public class ProtoMqttAdaptor implements MqttTransportAdaptor {
private static final ByteBufAllocator ALLOCATOR = new UnpooledByteBufAllocator(false);
@Override
public TransportProtos.PostTelemetryMsg convertToPostTelemetry(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException {
DeviceSessionCtx deviceSessionCtx = (DeviceSessionCtx) ctx;
@ -143,7 +136,6 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor {
}
}
@Override
public Optional<MqttMessage> convertToPublish(MqttDeviceAwareSessionContext ctx, TransportProtos.ToDeviceRpcRequestMsg rpcRequest) {
return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_RPC_REQUESTS_TOPIC + rpcRequest.getRequestId(), rpcRequest.toByteArray()));
@ -164,6 +156,11 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor {
return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC, provisionResponse.toByteArray()));
}
@Override
public Optional<MqttMessage> convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk) throws AdaptorException {
return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX + requestId + "/" + chunk, firmwareChunk));
}
@Override
public Optional<MqttMessage> convertToGatewayPublish(MqttDeviceAwareSessionContext ctx, String deviceName, TransportProtos.GetAttributeResponseMsg responseMsg) throws AdaptorException {
if (!StringUtils.isEmpty(responseMsg.getError())) {
@ -202,15 +199,6 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor {
return bytes;
}
private MqttPublishMessage createMqttPublishMsg(MqttDeviceAwareSessionContext ctx, String topic, byte[] payloadBytes) {
MqttFixedHeader mqttFixedHeader =
new MqttFixedHeader(MqttMessageType.PUBLISH, false, ctx.getQoSForTopic(topic), false, 0);
MqttPublishVariableHeader header = new MqttPublishVariableHeader(topic, ctx.nextMsgId());
ByteBuf payload = ALLOCATOR.buffer();
payload.writeBytes(payloadBytes);
return new MqttPublishMessage(mqttFixedHeader, header, payload);
}
private int getRequestId(String topicName, String topic) {
return Integer.parseInt(topicName.substring(topic.length()));
}

8
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java

@ -20,10 +20,9 @@ import lombok.Data;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cache.firmware.FirmwareCacheReader;
import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.queue.util.TbTransportComponent;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@ -51,6 +50,11 @@ public abstract class TransportContext {
@Getter
private ExecutorService executor;
@Getter
@Autowired
private FirmwareCacheReader firmwareCacheReader;
@PostConstruct
public void init() {
executor = Executors.newWorkStealingPool(50);

4
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java

@ -24,6 +24,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetFirmwareRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetFirmwareResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetResourceRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.GetResourceResponseMsg;
@ -98,6 +100,8 @@ public interface TransportService {
void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback<Void> callback);
void process(SessionInfoProto sessionInfoProto, GetFirmwareRequestMsg msg, TransportServiceCallback<GetFirmwareResponseMsg> callback);
SessionMetaData registerAsyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener);
SessionMetaData registerSyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout);

12
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java

@ -44,7 +44,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType;
import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg;
import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus;
import org.thingsboard.server.gen.transport.TransportProtos.ResponseStatus;
import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto;
import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateBasicMqttCredRequestMsg;
@ -423,12 +423,12 @@ public class JsonConverter {
private static JsonObject toJson(ProvisionDeviceResponseMsg payload, boolean toGateway, int requestId) {
JsonObject result = new JsonObject();
if (payload.getStatus() == TransportProtos.ProvisionResponseStatus.NOT_FOUND) {
if (payload.getStatus() == ResponseStatus.NOT_FOUND) {
result.addProperty("errorMsg", "Provision data was not found!");
result.addProperty("status", ProvisionResponseStatus.NOT_FOUND.name());
} else if (payload.getStatus() == TransportProtos.ProvisionResponseStatus.FAILURE) {
result.addProperty("status", ResponseStatus.NOT_FOUND.name());
} else if (payload.getStatus() == TransportProtos.ResponseStatus.FAILURE) {
result.addProperty("errorMsg", "Failed to provision device!");
result.addProperty("status", ProvisionResponseStatus.FAILURE.name());
result.addProperty("status", ResponseStatus.FAILURE.name());
} else {
if (toGateway) {
result.addProperty("id", requestId);
@ -445,7 +445,7 @@ public class JsonConverter {
break;
}
result.addProperty("credentialsType", payload.getCredentialsType().name());
result.addProperty("status", ProvisionResponseStatus.SUCCESS.name());
result.addProperty("status", ResponseStatus.SUCCESS.name());
}
return result;
}

18
common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java

@ -377,7 +377,6 @@ public class DefaultTransportService implements TransportService {
AsyncCallbackTemplate.withCallback(response, callback::onSuccess, callback::onError, transportCallbackExecutor);
}
@Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback<Void> callback) {
if (log.isTraceEnabled()) {
@ -529,6 +528,19 @@ public class DefaultTransportService implements TransportService {
}
}
@Override
public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetFirmwareRequestMsg msg, TransportServiceCallback<TransportProtos.GetFirmwareResponseMsg> callback) {
if (checkLimits(sessionInfo, msg, callback)) {
TbProtoQueueMsg<TransportProtos.TransportApiRequestMsg> protoMsg =
new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setFirmwareRequestMsg(msg).build());
AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), response -> {
TransportProtos.GetFirmwareResponseMsg firmwareResponseMsg = response.getValue().getFirmwareResponseMsg();
callback.onSuccess(firmwareResponseMsg);
}, callback::onError, transportCallbackExecutor);
}
}
@Override
public SessionMetaData reportActivity(TransportProtos.SessionInfoProto sessionInfo) {
return reportActivityInternal(sessionInfo);
@ -608,11 +620,11 @@ public class DefaultTransportService implements TransportService {
sessions.remove(toSessionId(sessionInfo));
}
private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback<Void> callback) {
private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback<?> callback) {
return checkLimits(sessionInfo, msg, callback, 0);
}
private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback<Void> callback, int dataPoints) {
private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback<?> callback, int dataPoints) {
if (log.isTraceEnabled()) {
log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg);
}

19
dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java

@ -20,6 +20,8 @@ import com.google.common.hash.Hashing;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.cache.Cache;
import org.springframework.cache.CacheManager;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Firmware;
import org.thingsboard.server.common.data.FirmwareInfo;
@ -34,8 +36,10 @@ import org.thingsboard.server.dao.service.PaginatedRemover;
import org.thingsboard.server.dao.tenant.TenantDao;
import java.nio.ByteBuffer;
import java.util.Collections;
import java.util.Optional;
import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE;
import static org.thingsboard.server.dao.service.Validator.validateId;
import static org.thingsboard.server.dao.service.Validator.validatePageLink;
@ -48,11 +52,13 @@ public class BaseFirmwareService implements FirmwareService {
private final TenantDao tenantDao;
private final FirmwareDao firmwareDao;
private final FirmwareInfoDao firmwareInfoDao;
private final CacheManager cacheManager;
public BaseFirmwareService(TenantDao tenantDao, FirmwareDao firmwareDao, FirmwareInfoDao firmwareInfoDao) {
public BaseFirmwareService(TenantDao tenantDao, FirmwareDao firmwareDao, FirmwareInfoDao firmwareInfoDao, CacheManager cacheManager) {
this.tenantDao = tenantDao;
this.firmwareDao = firmwareDao;
this.firmwareInfoDao = firmwareInfoDao;
this.cacheManager = cacheManager;
}
@Override
@ -60,6 +66,10 @@ public class BaseFirmwareService implements FirmwareService {
log.trace("Executing saveFirmwareInfo [{}]", firmwareInfo);
firmwareInfoValidator.validate(firmwareInfo, FirmwareInfo::getTenantId);
try {
FirmwareId firmwareId = firmwareInfo.getId();
if (firmwareId != null) {
cacheManager.getCache(FIRMWARE_CACHE).evict(firmwareId.toString());
}
return firmwareInfoDao.save(firmwareInfo.getTenantId(), firmwareInfo);
} catch (Exception t) {
ConstraintViolationException e = extractConstraintViolationException(t).orElse(null);
@ -76,6 +86,10 @@ public class BaseFirmwareService implements FirmwareService {
log.trace("Executing saveFirmware [{}]", firmware);
firmwareValidator.validate(firmware, FirmwareInfo::getTenantId);
try {
FirmwareId firmwareId = firmware.getId();
if (firmwareId != null) {
cacheManager.getCache(FIRMWARE_CACHE).evict(firmwareId.toString());
}
return firmwareDao.save(firmware.getTenantId(), firmware);
} catch (Exception t) {
ConstraintViolationException e = extractConstraintViolationException(t).orElse(null);
@ -122,6 +136,8 @@ public class BaseFirmwareService implements FirmwareService {
log.trace("Executing deleteFirmware [{}]", firmwareId);
validateId(firmwareId, INCORRECT_FIRMWARE_ID + firmwareId);
try {
Cache cache = cacheManager.getCache(FIRMWARE_CACHE);
cache.evict(Collections.singletonList(firmwareId));
firmwareDao.removeById(tenantId, firmwareId.getId());
} catch (Exception t) {
ConstraintViolationException e = extractConstraintViolationException(t).orElse(null);
@ -278,7 +294,6 @@ public class BaseFirmwareService implements FirmwareService {
private PaginatedRemover<TenantId, FirmwareInfo> tenantFirmwareRemover =
new PaginatedRemover<>() {
@Override
protected PageData<FirmwareInfo> findEntities(TenantId tenantId, TenantId id, PageLink pageLink) {
return firmwareInfoDao.findFirmwareInfoByTenantId(id, pageLink);

Loading…
Cancel
Save