diff --git a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java index 9613d708d2..768eb592f4 100644 --- a/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java +++ b/application/src/main/java/org/thingsboard/server/service/transport/DefaultTransportApiService.java @@ -23,16 +23,19 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.ByteString; import lombok.extern.slf4j.Slf4j; +import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cache.firmware.FirmwareCacheWriter; import org.thingsboard.server.common.data.ApiUsageState; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.ResourceType; +import org.thingsboard.server.common.data.TbResource; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.device.credentials.BasicMqttCredentials; import org.thingsboard.server.common.data.device.credentials.ProvisionDeviceCredentialsData; @@ -40,6 +43,7 @@ import org.thingsboard.server.common.data.device.profile.ProvisionDeviceProfileC import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.DeviceProfileId; +import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.security.DeviceCredentials; @@ -55,6 +59,7 @@ import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.provision.ProvisionFailedException; import org.thingsboard.server.dao.device.provision.ProvisionRequest; import org.thingsboard.server.dao.device.provision.ProvisionResponse; +import org.thingsboard.server.dao.firmware.FirmwareService; import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.dao.resource.TbResourceService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; @@ -66,7 +71,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFro import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceRequestMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.TransportApiResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; @@ -109,6 +113,8 @@ public class DefaultTransportApiService implements TransportApiService { private final DataDecodingEncodingService dataDecodingEncodingService; private final DeviceProvisionService deviceProvisionService; private final TbResourceService resourceService; + private final FirmwareService firmwareService; + private final FirmwareCacheWriter firmwareCacheWriter; private final ConcurrentMap deviceCreationLocks = new ConcurrentHashMap<>(); @@ -117,7 +123,7 @@ public class DefaultTransportApiService implements TransportApiService { RelationService relationService, DeviceCredentialsService deviceCredentialsService, DeviceStateService deviceStateService, DbCallbackExecutorService dbCallbackExecutorService, TbClusterService tbClusterService, DataDecodingEncodingService dataDecodingEncodingService, - DeviceProvisionService deviceProvisionService, TbResourceService resourceService) { + DeviceProvisionService deviceProvisionService, TbResourceService resourceService, FirmwareService firmwareService, CacheManager cacheManager, FirmwareCacheWriter firmwareCacheWriter) { this.deviceProfileCache = deviceProfileCache; this.tenantProfileCache = tenantProfileCache; this.apiUsageStateService = apiUsageStateService; @@ -130,6 +136,8 @@ public class DefaultTransportApiService implements TransportApiService { this.dataDecodingEncodingService = dataDecodingEncodingService; this.deviceProvisionService = deviceProvisionService; this.resourceService = resourceService; + this.firmwareService = firmwareService; + this.firmwareCacheWriter = firmwareCacheWriter; } @Override @@ -166,6 +174,9 @@ public class DefaultTransportApiService implements TransportApiService { } else if (transportApiRequestMsg.hasResourceRequestMsg()) { return Futures.transform(handle(transportApiRequestMsg.getResourceRequestMsg()), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); + } else if (transportApiRequestMsg.hasFirmwareRequestMsg()) { + return Futures.transform(handle(transportApiRequestMsg.getFirmwareRequestMsg()), + value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); } return Futures.transform(getEmptyTransportApiResponseFuture(), value -> new TbProtoQueueMsg<>(tbProtoQueueMsg.getKey(), value, tbProtoQueueMsg.getHeaders()), MoreExecutors.directExecutor()); @@ -313,14 +324,14 @@ public class DefaultTransportApiService implements TransportApiService { } catch (ProvisionFailedException e) { return Futures.immediateFuture(getTransportApiResponseMsg( new DeviceCredentials(), - TransportProtos.ProvisionResponseStatus.valueOf(e.getMessage()))); + TransportProtos.ResponseStatus.valueOf(e.getMessage()))); } - return Futures.transform(provisionResponseFuture, provisionResponse -> getTransportApiResponseMsg(provisionResponse.getDeviceCredentials(), TransportProtos.ProvisionResponseStatus.SUCCESS), + return Futures.transform(provisionResponseFuture, provisionResponse -> getTransportApiResponseMsg(provisionResponse.getDeviceCredentials(), TransportProtos.ResponseStatus.SUCCESS), dbCallbackExecutorService); } - private TransportApiResponseMsg getTransportApiResponseMsg(DeviceCredentials deviceCredentials, TransportProtos.ProvisionResponseStatus status) { - if (!status.equals(ProvisionResponseStatus.SUCCESS)) { + private TransportApiResponseMsg getTransportApiResponseMsg(DeviceCredentials deviceCredentials, TransportProtos.ResponseStatus status) { + if (!status.equals(TransportProtos.ResponseStatus.SUCCESS)) { return TransportApiResponseMsg.newBuilder().setProvisionDeviceResponseMsg(TransportProtos.ProvisionDeviceResponseMsg.newBuilder().setStatus(status).build()).build(); } TransportProtos.ProvisionDeviceResponseMsg.Builder provisionResponse = TransportProtos.ProvisionDeviceResponseMsg.newBuilder() @@ -438,6 +449,46 @@ public class DefaultTransportApiService implements TransportApiService { } } + private ListenableFuture handle(TransportProtos.GetFirmwareRequestMsg requestMsg) { + TenantId tenantId = new TenantId(new UUID(requestMsg.getTenantIdMSB(), requestMsg.getTenantIdLSB())); + DeviceId deviceId = new DeviceId(new UUID(requestMsg.getDeviceIdMSB(), requestMsg.getDeviceIdLSB())); + Device device = deviceService.findDeviceById(tenantId, deviceId); + + if (device == null) { + return getEmptyTransportApiResponseFuture(); + } + + FirmwareId firmwareId = device.getFirmwareId(); + + if (firmwareId == null) { + firmwareId = deviceProfileCache.find(device.getDeviceProfileId()).getFirmwareId(); + } + + TransportProtos.GetFirmwareResponseMsg.Builder builder = TransportProtos.GetFirmwareResponseMsg.newBuilder(); + + if (firmwareId == null) { + builder.setResponseStatus(TransportProtos.ResponseStatus.NOT_FOUND); + } else { + Firmware firmware = firmwareService.findFirmwareById(tenantId, firmwareId); + + if (firmware == null) { + builder.setResponseStatus(TransportProtos.ResponseStatus.NOT_FOUND); + } else { + builder.setResponseStatus(TransportProtos.ResponseStatus.SUCCESS); + builder.setFirmwareIdMSB(firmwareId.getId().getMostSignificantBits()); + builder.setFirmwareIdLSB(firmwareId.getId().getLeastSignificantBits()); + builder.setFileName(firmware.getFileName()); + builder.setContentType(firmware.getContentType()); + firmwareCacheWriter.put(firmwareId.toString(), firmware.getData().array()); + } + } + + return Futures.immediateFuture( + TransportApiResponseMsg.newBuilder() + .setFirmwareResponseMsg(builder.build()) + .build()); + } + private ListenableFuture handleRegistration(TransportProtos.LwM2MRegistrationRequestMsg msg) { TenantId tenantId = new TenantId(UUID.fromString(msg.getTenantId())); String deviceName = msg.getEndpoint(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 01b2eb47b6..777d75974f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -281,7 +281,7 @@ actors: tenant: create_components_on_init: "${ACTORS_TENANT_CREATE_COMPONENTS_ON_INIT:true}" session: - max_concurrent_sessions_per_device: "${ACTORS_MAX_CONCURRENT_SESSION_PER_DEVICE:1}" + max_concurrent_sessions_per_device: "${ACTORS_MAX_CONCURRENT_SESSION_PER_DEVICE:2}" sync: # Default timeout for processing request using synchronous session (HTTP, CoAP) in milliseconds timeout: "${ACTORS_SESSION_SYNC_TIMEOUT:10000}" @@ -365,6 +365,9 @@ caffeine: tokensOutdatageTime: timeToLiveInMinutes: 20000 maxSize: 10000 + firmwares: + timeToLiveInMinutes: 1440 + maxSize: 100 redis: # standalone or cluster diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/firmware/AbstractRedisFirmwareCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/AbstractRedisFirmwareCache.java new file mode 100644 index 0000000000..371b85b94d --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/AbstractRedisFirmwareCache.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.firmware; + +import org.springframework.data.redis.connection.RedisConnectionFactory; + +import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE; + +public abstract class AbstractRedisFirmwareCache { + + protected final RedisConnectionFactory redisConnectionFactory; + + protected AbstractRedisFirmwareCache(RedisConnectionFactory redisConnectionFactory) { + this.redisConnectionFactory = redisConnectionFactory; + } + + protected byte[] toFirmwareCacheKey(String key) { + return String.format("%s::%s", FIRMWARE_CACHE, key).getBytes(); + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheReader.java b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheReader.java new file mode 100644 index 0000000000..f7e05f818c --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheReader.java @@ -0,0 +1,60 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.firmware; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; + +import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE; + +@Service +@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport') && ('${cache.type:null}'=='caffeine' || '${cache.type:null}'=='caffeine')") +public class CaffeineFirmwareCacheReader implements FirmwareCacheReader { + + private final CacheManager cacheManager; + + public CaffeineFirmwareCacheReader(CacheManager cacheManager) { + this.cacheManager = cacheManager; + } + + @Override + public byte[] get(String key) { + return get(key, 0, 0); + } + + @Override + public byte[] get(String key, int chunkSize, int chunk) { + byte[] data = cacheManager.getCache(FIRMWARE_CACHE).get(key, byte[].class); + + if (chunkSize < 1) { + return data; + } + + if (data != null && data.length > 0) { + int startIndex = chunkSize * chunk; + + int size = Math.min(data.length - startIndex, chunkSize); + + if (startIndex < data.length && size > 0) { + byte[] result = new byte[size]; + System.arraycopy(data, startIndex, result, 0, size); + return result; + } + } + return new byte[0]; + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheWriter.java b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheWriter.java new file mode 100644 index 0000000000..e543739382 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/CaffeineFirmwareCacheWriter.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.firmware; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.cache.CacheManager; +import org.springframework.stereotype.Service; + +import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE; + +@Service +@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='core') && ('${cache.type:null}'=='caffeine' || '${cache.type:null}'=='caffeine')") +public class CaffeineFirmwareCacheWriter implements FirmwareCacheWriter { + + private final CacheManager cacheManager; + + public CaffeineFirmwareCacheWriter(CacheManager cacheManager) { + this.cacheManager = cacheManager; + } + + @Override + public void put(String key, byte[] value) { + cacheManager.getCache(FIRMWARE_CACHE).putIfAbsent(key, value); + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheReader.java b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheReader.java new file mode 100644 index 0000000000..4b4dab4caa --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheReader.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.firmware; + +public interface FirmwareCacheReader { + byte[] get(String key); + + byte[] get(String key, int chunkSize, int chunk); +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheWriter.java b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheWriter.java new file mode 100644 index 0000000000..f1387f9ff1 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/FirmwareCacheWriter.java @@ -0,0 +1,20 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.firmware; + +public interface FirmwareCacheWriter { + void put(String key, byte[] value); +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheReader.java b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheReader.java new file mode 100644 index 0000000000..9bf8b40230 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheReader.java @@ -0,0 +1,49 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.firmware; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; + +@Service +@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='tb-transport') && '${cache.type:null}'=='redis'") +public class RedisFirmwareCacheReader extends AbstractRedisFirmwareCache implements FirmwareCacheReader { + + public RedisFirmwareCacheReader(RedisConnectionFactory redisConnectionFactory) { + super(redisConnectionFactory); + } + + @Override + public byte[] get(String key) { + return get(key, 0, 0); + } + + @Override + public byte[] get(String key, int chunkSize, int chunk) { + try (RedisConnection connection = redisConnectionFactory.getConnection()) { + if (chunkSize == 0) { + return connection.get(toFirmwareCacheKey(key)); + } + + int startIndex = chunkSize * chunk; + int endIndex = startIndex + chunkSize - 1; + return connection.getRange(toFirmwareCacheKey(key), startIndex, endIndex); + } + } + +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheWriter.java b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheWriter.java new file mode 100644 index 0000000000..6ba3ee91f6 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/firmware/RedisFirmwareCacheWriter.java @@ -0,0 +1,38 @@ +/** + * Copyright © 2016-2021 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache.firmware; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.stereotype.Service; + +@Service +@ConditionalOnExpression("(('${service.type:null}'=='monolith' && '${transport.api_enabled:true}'=='true') || '${service.type:null}'=='core') && '${cache.type:null}'=='redis'") +public class RedisFirmwareCacheWriter extends AbstractRedisFirmwareCache implements FirmwareCacheWriter { + + public RedisFirmwareCacheWriter(RedisConnectionFactory redisConnectionFactory) { + super(redisConnectionFactory); + } + + @Override + public void put(String key, byte[] value) { + try (RedisConnection connection = redisConnectionFactory.getConnection()) { + connection.set(toFirmwareCacheKey(key), value); + } + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java index 62f625f285..ba14b24292 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/CacheConstants.java @@ -28,4 +28,5 @@ public class CacheConstants { public static final String DEVICE_PROFILE_CACHE = "deviceProfiles"; public static final String ATTRIBUTES_CACHE = "attributes"; public static final String TOKEN_OUTDATAGE_TIME_CACHE = "tokensOutdatageTime"; + public static final String FIRMWARE_CACHE = "firmwares"; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java index f736fe371f..419f9c7f43 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/device/profile/MqttTopics.java @@ -30,6 +30,9 @@ public class MqttTopics { private static final String CLAIM = "/claim"; private static final String SUB_TOPIC = "+"; private static final String PROVISION = "/provision"; + private static final String FIRMWARE = "/fw"; + private static final String CHUNK = "/chunk/"; + private static final String ERROR = "/error"; private static final String ATTRIBUTES_RESPONSE = ATTRIBUTES + RESPONSE; private static final String ATTRIBUTES_REQUEST = ATTRIBUTES + REQUEST; @@ -69,6 +72,13 @@ public class MqttTopics { public static final String GATEWAY_ATTRIBUTES_REQUEST_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_REQUEST; public static final String GATEWAY_ATTRIBUTES_RESPONSE_TOPIC = BASE_GATEWAY_API_TOPIC + ATTRIBUTES_RESPONSE; + // v2 topics + public static final String BASE_DEVICE_API_TOPIC_V2 = "v2"; + + public static final String DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + RESPONSE + "/"; + public static final String DEVICE_FIRMWARE_RESPONSES_TOPIC = DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX + SUB_TOPIC + CHUNK + SUB_TOPIC; + public static final String DEVICE_FIRMWARE_ERROR_TOPIC = BASE_DEVICE_API_TOPIC_V2 + FIRMWARE + ERROR; + private MqttTopics() { } } diff --git a/common/queue/src/main/proto/queue.proto b/common/queue/src/main/proto/queue.proto index 7e9ff076f2..7abcd3613f 100644 --- a/common/queue/src/main/proto/queue.proto +++ b/common/queue/src/main/proto/queue.proto @@ -336,17 +336,33 @@ message ProvisionDeviceCredentialsMsg { } message ProvisionDeviceResponseMsg { - ProvisionResponseStatus status = 1; + ResponseStatus status = 1; CredentialsType credentialsType = 2; string credentialsValue = 3; } -enum ProvisionResponseStatus { +enum ResponseStatus { UNKNOWN = 0; SUCCESS = 1; NOT_FOUND = 2; FAILURE = 3; } + +message GetFirmwareRequestMsg { + int64 deviceIdMSB = 1; + int64 deviceIdLSB = 2; + int64 tenantIdMSB = 3; + int64 tenantIdLSB = 4; +} + +message GetFirmwareResponseMsg { + ResponseStatus responseStatus = 1; + int64 firmwareIdMSB = 2; + int64 firmwareIdLSB = 3; + string contentType = 4; + string fileName = 5; +} + //Used to report session state to tb-Service and persist this state in the cache on the tb-Service level. message SubscriptionInfoProto { int64 lastActivityTime = 1; @@ -552,6 +568,7 @@ message TransportApiRequestMsg { ProvisionDeviceRequestMsg provisionDeviceRequestMsg = 7; ValidateDeviceLwM2MCredentialsRequestMsg validateDeviceLwM2MCredentialsRequestMsg = 8; GetResourceRequestMsg resourceRequestMsg = 9; + GetFirmwareRequestMsg firmwareRequestMsg = 10; } /* Response from ThingsBoard Core Service to Transport Service */ @@ -562,6 +579,7 @@ message TransportApiResponseMsg { ProvisionDeviceResponseMsg provisionDeviceResponseMsg = 4; LwM2MResponseMsg lwM2MResponseMsg = 6; GetResourceResponseMsg resourceResponseMsg = 7; + GetFirmwareResponseMsg firmwareResponseMsg = 8; } /* Messages that are handled by ThingsBoard Core Service */ diff --git a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java index c1fd5b6a4a..89e5213fd0 100644 --- a/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java +++ b/common/transport/coap/src/main/java/org/thingsboard/server/transport/coap/CoapTransportResource.java @@ -360,7 +360,7 @@ public class CoapTransportResource extends AbstractCoapTransportResource { @Override public void onSuccess(TransportProtos.ProvisionDeviceResponseMsg msg) { CoAP.ResponseCode responseCode = CoAP.ResponseCode.CREATED; - if (!msg.getStatus().equals(TransportProtos.ProvisionResponseStatus.SUCCESS)) { + if (!msg.getStatus().equals(TransportProtos.ResponseStatus.SUCCESS)) { responseCode = CoAP.ResponseCode.BAD_REQUEST; } if (payloadType.equals(TransportPayloadType.JSON)) { diff --git a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java index e222b690ef..5c10202d00 100644 --- a/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java +++ b/common/transport/http/src/main/java/org/thingsboard/server/transport/http/DeviceApiController.java @@ -20,7 +20,10 @@ import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.PathVariable; @@ -41,7 +44,6 @@ import org.thingsboard.server.common.transport.auth.SessionInfoCreator; import org.thingsboard.server.common.transport.auth.ValidateDeviceCredentialsResponse; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; -import org.thingsboard.server.gen.transport.TransportProtos.DeviceInfoProto; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; @@ -53,7 +55,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcRequestMs import org.thingsboard.server.gen.transport.TransportProtos.ToDeviceRpcResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ToServerRpcResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceCredentialsResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import javax.servlet.http.HttpServletRequest; @@ -204,6 +205,23 @@ public class DeviceApiController { return responseWriter; } + @RequestMapping(value = "/{deviceToken}/firmware", method = RequestMethod.GET) + public DeferredResult getFirmware(@PathVariable("deviceToken") String deviceToken, + @RequestParam(value = "chunkSize", required = false, defaultValue = "0") int chunkSize, + @RequestParam(value = "chunk", required = false, defaultValue = "0") int chunk) { + DeferredResult responseWriter = new DeferredResult<>(); + transportContext.getTransportService().process(DeviceTransportType.DEFAULT, ValidateDeviceTokenRequestMsg.newBuilder().setToken(deviceToken).build(), + new DeviceAuthCallback(transportContext, responseWriter, sessionInfo -> { + TransportProtos.GetFirmwareRequestMsg requestMsg = TransportProtos.GetFirmwareRequestMsg.newBuilder() + .setTenantIdMSB(sessionInfo.getTenantIdMSB()) + .setTenantIdLSB(sessionInfo.getTenantIdLSB()) + .setDeviceIdMSB(sessionInfo.getDeviceIdMSB()) + .setDeviceIdLSB(sessionInfo.getDeviceIdLSB()).build(); + transportContext.getTransportService().process(sessionInfo, requestMsg, new GetFirmwareCallback(responseWriter, chunkSize, chunk)); + })); + return responseWriter; + } + @RequestMapping(value = "/provision", method = RequestMethod.POST) public DeferredResult provisionDevice(@RequestBody String json, HttpServletRequest httpRequest) { DeferredResult responseWriter = new DeferredResult<>(); @@ -258,6 +276,41 @@ public class DeviceApiController { } } + private class GetFirmwareCallback implements TransportServiceCallback { + private final DeferredResult responseWriter; + private final int chuckSize; + private final int chuck; + + GetFirmwareCallback(DeferredResult responseWriter, int chuckSize, int chuck) { + this.responseWriter = responseWriter; + this.chuckSize = chuckSize; + this.chuck = chuck; + } + + @Override + public void onSuccess(TransportProtos.GetFirmwareResponseMsg firmwareResponseMsg) { + if (!TransportProtos.ResponseStatus.SUCCESS.equals(firmwareResponseMsg.getResponseStatus())) { + responseWriter.setResult(new ResponseEntity<>(HttpStatus.NOT_FOUND)); + } else { + String firmwareId = new UUID(firmwareResponseMsg.getFirmwareIdMSB(), firmwareResponseMsg.getFirmwareIdLSB()).toString(); + ByteArrayResource resource = new ByteArrayResource(transportContext.getFirmwareCacheReader().get(firmwareId, chuckSize, chuck)); + ResponseEntity response = ResponseEntity.ok() + .header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + firmwareResponseMsg.getFileName()) + .header("x-filename", firmwareResponseMsg.getFileName()) + .contentLength(resource.contentLength()) + .contentType(parseMediaType(firmwareResponseMsg.getContentType())) + .body(resource); + responseWriter.setResult(response); + } + } + + @Override + public void onError(Throwable e) { + log.warn("Failed to process request", e); + responseWriter.setResult(new ResponseEntity<>(HttpStatus.INTERNAL_SERVER_ERROR)); + } + } + private static class SessionCloseOnErrorCallback implements TransportServiceCallback { private final TransportService transportService; private final SessionInfoProto sessionInfo; @@ -338,4 +391,12 @@ public class DeviceApiController { .build(), TransportServiceCallback.EMPTY); } + private static MediaType parseMediaType(String contentType) { + try { + return MediaType.parseMediaType(contentType); + } catch (Exception e) { + return MediaType.APPLICATION_OCTET_STREAM; + } + } + } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisSecurityStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisSecurityStore.java index 7d05630ba4..fa0cf67694 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisSecurityStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisSecurityStore.java @@ -29,7 +29,6 @@ import redis.clients.jedis.ScanResult; import java.util.Collection; import java.util.LinkedList; -@Service public class TbLwM2mRedisSecurityStore implements EditableSecurityStore { private static final String SEC_EP = "SEC#EP#"; diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index 06a8dcacdc..de525d107d 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -46,6 +46,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.MqttTopics; +import org.thingsboard.server.common.data.id.FirmwareId; import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; import org.thingsboard.server.common.transport.SessionMsgListener; @@ -69,10 +70,10 @@ import org.thingsboard.server.transport.mqtt.session.MqttTopicMatcher; import org.thingsboard.server.transport.mqtt.util.SslUtil; import javax.net.ssl.SSLPeerUnverifiedException; -import java.security.cert.Certificate; -import java.security.cert.X509Certificate; import java.io.IOException; import java.net.InetSocketAddress; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -80,7 +81,10 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import static com.amazonaws.util.StringUtils.UTF8; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_ACCEPTED; import static io.netty.handler.codec.mqtt.MqttConnectReturnCode.CONNECTION_REFUSED_NOT_AUTHORIZED; import static io.netty.handler.codec.mqtt.MqttMessageType.CONNACK; @@ -99,6 +103,10 @@ import static io.netty.handler.codec.mqtt.MqttQoS.FAILURE; @Slf4j public class MqttTransportHandler extends ChannelInboundHandlerAdapter implements GenericFutureListener>, SessionMsgListener { + private static final Pattern FW_PATTERN = Pattern.compile("v2/fw/request/(?\\d+)/chunk/(?\\d+)"); + + private static final String PAYLOAD_TOO_LARGE = "PAYLOAD_TOO_LARGE"; + private static final MqttQoS MAX_SUPPORTED_QOS_LVL = AT_LEAST_ONCE; private final UUID sessionId; @@ -112,6 +120,9 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private volatile InetSocketAddress address; private volatile GatewaySessionHandler gatewaySessionHandler; + private final ConcurrentHashMap fwSessions; + private final ConcurrentHashMap fwChunkSizes; + MqttTransportHandler(MqttTransportContext context, SslHandler sslHandler) { this.sessionId = UUID.randomUUID(); this.context = context; @@ -120,6 +131,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.sslHandler = sslHandler; this.mqttQoSMap = new ConcurrentHashMap<>(); this.deviceSessionCtx = new DeviceSessionCtx(sessionId, mqttQoSMap, context); + this.fwSessions = new ConcurrentHashMap<>(); + this.fwChunkSizes = new ConcurrentHashMap<>(); } @Override @@ -280,6 +293,7 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement private void processDevicePublish(ChannelHandlerContext ctx, MqttPublishMessage mqttMsg, String topicName, int msgId) { try { + Matcher fwMatcher; MqttTransportAdaptor payloadAdaptor = deviceSessionCtx.getPayloadAdaptor(); if (deviceSessionCtx.isDeviceAttributesTopic(topicName)) { TransportProtos.PostAttributeMsg postAttributeMsg = payloadAdaptor.convertToPostAttributes(deviceSessionCtx, mqttMsg); @@ -299,6 +313,38 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } else if (topicName.equals(MqttTopics.DEVICE_CLAIM_TOPIC)) { TransportProtos.ClaimDeviceMsg claimDeviceMsg = payloadAdaptor.convertToClaimDevice(deviceSessionCtx, mqttMsg); transportService.process(deviceSessionCtx.getSessionInfo(), claimDeviceMsg, getPubAckCallback(ctx, msgId, claimDeviceMsg)); + } else if ((fwMatcher = FW_PATTERN.matcher(topicName)).find()) { + String payload = mqttMsg.content().toString(UTF8); + int chunkSize = payload != null ? Integer.parseInt(payload) : 0; + String requestId = fwMatcher.group("requestId"); + int chunk = Integer.parseInt(fwMatcher.group("chunk")); + + if (chunkSize > 0) { + this.fwChunkSizes.put(requestId, chunkSize); + } else { + chunkSize = fwChunkSizes.getOrDefault(requestId, 0); + } + + if (chunkSize > context.getMaxPayloadSize()) { + sendFirmwareError(ctx, PAYLOAD_TOO_LARGE); + return; + } + + String firmwareId = fwSessions.get(requestId); + + if (firmwareId != null) { + sendFirmware(ctx, mqttMsg.variableHeader().packetId(), firmwareId, requestId, chunkSize, chunk); + } else { + TransportProtos.SessionInfoProto sessionInfo = deviceSessionCtx.getSessionInfo(); + TransportProtos.GetFirmwareRequestMsg getFirmwareRequestMsg = TransportProtos.GetFirmwareRequestMsg.newBuilder() + .setDeviceIdMSB(sessionInfo.getDeviceIdMSB()) + .setDeviceIdLSB(sessionInfo.getDeviceIdLSB()) + .setTenantIdMSB(sessionInfo.getTenantIdMSB()) + .setTenantIdLSB(sessionInfo.getTenantIdLSB()) + .build(); + transportService.process(deviceSessionCtx.getSessionInfo(), getFirmwareRequestMsg, + new FirmwareCallback(ctx, mqttMsg.variableHeader().packetId(), getFirmwareRequestMsg, requestId, chunkSize, chunk)); + } } else { transportService.reportActivity(deviceSessionCtx.getSessionInfo()); ack(ctx, msgId); @@ -366,6 +412,65 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } } + private class FirmwareCallback implements TransportServiceCallback { + private final ChannelHandlerContext ctx; + private final int msgId; + private final TransportProtos.GetFirmwareRequestMsg msg; + private final String requestId; + private final int chunkSize; + private final int chunk; + + FirmwareCallback(ChannelHandlerContext ctx, int msgId, TransportProtos.GetFirmwareRequestMsg msg, String requestId, int chunkSize, int chunk) { + this.ctx = ctx; + this.msgId = msgId; + this.msg = msg; + this.requestId = requestId; + this.chunkSize = chunkSize; + this.chunk = chunk; + } + + @Override + public void onSuccess(TransportProtos.GetFirmwareResponseMsg response) { + if (TransportProtos.ResponseStatus.SUCCESS.equals(response.getResponseStatus())) { + FirmwareId firmwareId = new FirmwareId(new UUID(response.getFirmwareIdMSB(), response.getFirmwareIdLSB())); + fwSessions.put(requestId, firmwareId.toString()); + sendFirmware(ctx, msgId, firmwareId.toString(), requestId, chunkSize, chunk); + } else { + sendFirmwareError(ctx, response.getResponseStatus().toString()); + } + } + + @Override + public void onError(Throwable e) { + log.trace("[{}] Failed to get firmware: {}", sessionId, msg, e); + processDisconnect(ctx); + } + } + + private void sendFirmware(ChannelHandlerContext ctx, int msgId, String firmwareId, String requestId, int chunkSize, int chunk) { + log.trace("[{}] Send firmware [{}] to device!", sessionId, firmwareId); + ack(ctx, msgId); + try { + byte[] firmwareChunk = context.getFirmwareCacheReader().get(firmwareId, chunkSize, chunk); + deviceSessionCtx.getPayloadAdaptor() + .convertToPublish(deviceSessionCtx, firmwareChunk, requestId, chunk) + .ifPresent(deviceSessionCtx.getChannel()::writeAndFlush); + if (firmwareChunk != null && chunkSize != firmwareChunk.length) { + scheduler.schedule(() -> processDisconnect(ctx), 60, TimeUnit.SECONDS); + } + } catch (Exception e) { + log.trace("[{}] Failed to send firmware response!", sessionId, e); + } + } + + private void sendFirmwareError(ChannelHandlerContext ctx, String error) { + log.warn("[{}] {}", sessionId, error); + deviceSessionCtx.getChannel().writeAndFlush(deviceSessionCtx + .getPayloadAdaptor() + .createMqttPublishMsg(deviceSessionCtx, MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC, error.getBytes())); + processDisconnect(ctx); + } + private void processSubscribe(ChannelHandlerContext ctx, MqttSubscribeMessage mqttMsg) { if (!checkConnected(ctx, mqttMsg)) { return; @@ -396,6 +501,8 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement case MqttTopics.GATEWAY_RPC_TOPIC: case MqttTopics.GATEWAY_ATTRIBUTES_RESPONSE_TOPIC: case MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC: + case MqttTopics.DEVICE_FIRMWARE_RESPONSES_TOPIC: + case MqttTopics.DEVICE_FIRMWARE_ERROR_TOPIC: registerSubQoS(topic, grantedQoSList, reqQoS); break; default: diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java index 47a3a9fe87..c7690b756f 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/JsonMqttAdaptor.java @@ -21,8 +21,6 @@ import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonSyntaxException; import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.UnpooledByteBufAllocator; import io.netty.handler.codec.mqtt.MqttFixedHeader; import io.netty.handler.codec.mqtt.MqttMessage; import io.netty.handler.codec.mqtt.MqttMessageType; @@ -31,10 +29,10 @@ import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.common.transport.adaptor.JsonConverter; import org.thingsboard.server.gen.transport.TransportProtos; -import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.transport.mqtt.session.MqttDeviceAwareSessionContext; import java.nio.charset.Charset; @@ -55,7 +53,6 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { protected static final Charset UTF8 = StandardCharsets.UTF_8; private static final Gson GSON = new Gson(); - private static final ByteBufAllocator ALLOCATOR = new UnpooledByteBufAllocator(false); @Override public TransportProtos.PostTelemetryMsg convertToPostTelemetry(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException { @@ -153,6 +150,11 @@ public class JsonMqttAdaptor implements MqttTransportAdaptor { return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC, JsonConverter.toJson(provisionResponse))); } + @Override + public Optional convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk) { + return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX + requestId + "/" + chunk, firmwareChunk)); + } + public static JsonElement validateJsonPayload(UUID sessionId, ByteBuf payloadData) throws AdaptorException { String payload = validatePayload(sessionId, payloadData, false); try { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java index e8a455a923..2bd35df8d9 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/MqttTransportAdaptor.java @@ -15,8 +15,14 @@ */ package org.thingsboard.server.transport.mqtt.adaptors; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.handler.codec.mqtt.MqttFixedHeader; import io.netty.handler.codec.mqtt.MqttMessage; +import io.netty.handler.codec.mqtt.MqttMessageType; import io.netty.handler.codec.mqtt.MqttPublishMessage; +import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; import org.thingsboard.server.common.transport.adaptor.AdaptorException; import org.thingsboard.server.gen.transport.TransportProtos.AttributeUpdateNotificationMsg; import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; @@ -39,6 +45,8 @@ import java.util.Optional; */ public interface MqttTransportAdaptor { + ByteBufAllocator ALLOCATOR = new UnpooledByteBufAllocator(false); + PostTelemetryMsg convertToPostTelemetry(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException; PostAttributeMsg convertToPostAttributes(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException; @@ -69,4 +77,14 @@ public interface MqttTransportAdaptor { Optional convertToPublish(MqttDeviceAwareSessionContext ctx, ProvisionDeviceResponseMsg provisionResponse) throws AdaptorException; + Optional convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk) throws AdaptorException; + + default MqttPublishMessage createMqttPublishMsg(MqttDeviceAwareSessionContext ctx, String topic, byte[] payloadInBytes) { + MqttFixedHeader mqttFixedHeader = + new MqttFixedHeader(MqttMessageType.PUBLISH, false, ctx.getQoSForTopic(topic), false, 0); + MqttPublishVariableHeader header = new MqttPublishVariableHeader(topic, ctx.nextMsgId()); + ByteBuf payload = ALLOCATOR.buffer(); + payload.writeBytes(payloadInBytes); + return new MqttPublishMessage(mqttFixedHeader, header, payload); + } } diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java index a004fbd500..82287bfdce 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/adaptors/ProtoMqttAdaptor.java @@ -21,13 +21,8 @@ import com.google.protobuf.DynamicMessage; import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.util.JsonFormat; import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufAllocator; -import io.netty.buffer.UnpooledByteBufAllocator; -import io.netty.handler.codec.mqtt.MqttFixedHeader; import io.netty.handler.codec.mqtt.MqttMessage; -import io.netty.handler.codec.mqtt.MqttMessageType; import io.netty.handler.codec.mqtt.MqttPublishMessage; -import io.netty.handler.codec.mqtt.MqttPublishVariableHeader; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; @@ -46,8 +41,6 @@ import java.util.Optional; @Slf4j public class ProtoMqttAdaptor implements MqttTransportAdaptor { - private static final ByteBufAllocator ALLOCATOR = new UnpooledByteBufAllocator(false); - @Override public TransportProtos.PostTelemetryMsg convertToPostTelemetry(MqttDeviceAwareSessionContext ctx, MqttPublishMessage inbound) throws AdaptorException { DeviceSessionCtx deviceSessionCtx = (DeviceSessionCtx) ctx; @@ -143,7 +136,6 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor { } } - @Override public Optional convertToPublish(MqttDeviceAwareSessionContext ctx, TransportProtos.ToDeviceRpcRequestMsg rpcRequest) { return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_RPC_REQUESTS_TOPIC + rpcRequest.getRequestId(), rpcRequest.toByteArray())); @@ -164,6 +156,11 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor { return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_PROVISION_RESPONSE_TOPIC, provisionResponse.toByteArray())); } + @Override + public Optional convertToPublish(MqttDeviceAwareSessionContext ctx, byte[] firmwareChunk, String requestId, int chunk) throws AdaptorException { + return Optional.of(createMqttPublishMsg(ctx, MqttTopics.DEVICE_FIRMWARE_RESPONSE_TOPIC_PREFIX + requestId + "/" + chunk, firmwareChunk)); + } + @Override public Optional convertToGatewayPublish(MqttDeviceAwareSessionContext ctx, String deviceName, TransportProtos.GetAttributeResponseMsg responseMsg) throws AdaptorException { if (!StringUtils.isEmpty(responseMsg.getError())) { @@ -202,15 +199,6 @@ public class ProtoMqttAdaptor implements MqttTransportAdaptor { return bytes; } - private MqttPublishMessage createMqttPublishMsg(MqttDeviceAwareSessionContext ctx, String topic, byte[] payloadBytes) { - MqttFixedHeader mqttFixedHeader = - new MqttFixedHeader(MqttMessageType.PUBLISH, false, ctx.getQoSForTopic(topic), false, 0); - MqttPublishVariableHeader header = new MqttPublishVariableHeader(topic, ctx.nextMsgId()); - ByteBuf payload = ALLOCATOR.buffer(); - payload.writeBytes(payloadBytes); - return new MqttPublishMessage(mqttFixedHeader, header, payload); - } - private int getRequestId(String topicName, String topic) { return Integer.parseInt(topicName.substring(topic.length())); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java index 75f20cc825..a1d09b78a7 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportContext.java @@ -20,10 +20,9 @@ import lombok.Data; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; +import org.thingsboard.server.cache.firmware.FirmwareCacheReader; import org.thingsboard.server.queue.discovery.TbServiceInfoProvider; import org.thingsboard.server.queue.scheduler.SchedulerComponent; -import org.thingsboard.server.queue.util.TbTransportComponent; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; @@ -51,6 +50,11 @@ public abstract class TransportContext { @Getter private ExecutorService executor; + + @Getter + @Autowired + private FirmwareCacheReader firmwareCacheReader; + @PostConstruct public void init() { executor = Executors.newWorkStealingPool(50); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 04b08272b1..c851be2586 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -24,6 +24,8 @@ import org.thingsboard.server.gen.transport.TransportProtos.ClaimDeviceMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetAttributeRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetEntityProfileResponseMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetFirmwareRequestMsg; +import org.thingsboard.server.gen.transport.TransportProtos.GetFirmwareResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetOrCreateDeviceFromGatewayRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourceRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.GetResourceResponseMsg; @@ -98,6 +100,8 @@ public interface TransportService { void process(SessionInfoProto sessionInfo, ClaimDeviceMsg msg, TransportServiceCallback callback); + void process(SessionInfoProto sessionInfoProto, GetFirmwareRequestMsg msg, TransportServiceCallback callback); + SessionMetaData registerAsyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener); SessionMetaData registerSyncSession(SessionInfoProto sessionInfo, SessionMsgListener listener, long timeout); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java index 2c5dfafe4e..680ad0e837 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/adaptor/JsonConverter.java @@ -44,7 +44,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; import org.thingsboard.server.gen.transport.TransportProtos.PostAttributeMsg; import org.thingsboard.server.gen.transport.TransportProtos.PostTelemetryMsg; import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; -import org.thingsboard.server.gen.transport.TransportProtos.ProvisionResponseStatus; +import org.thingsboard.server.gen.transport.TransportProtos.ResponseStatus; import org.thingsboard.server.gen.transport.TransportProtos.TsKvListProto; import org.thingsboard.server.gen.transport.TransportProtos.TsKvProto; import org.thingsboard.server.gen.transport.TransportProtos.ValidateBasicMqttCredRequestMsg; @@ -423,12 +423,12 @@ public class JsonConverter { private static JsonObject toJson(ProvisionDeviceResponseMsg payload, boolean toGateway, int requestId) { JsonObject result = new JsonObject(); - if (payload.getStatus() == TransportProtos.ProvisionResponseStatus.NOT_FOUND) { + if (payload.getStatus() == ResponseStatus.NOT_FOUND) { result.addProperty("errorMsg", "Provision data was not found!"); - result.addProperty("status", ProvisionResponseStatus.NOT_FOUND.name()); - } else if (payload.getStatus() == TransportProtos.ProvisionResponseStatus.FAILURE) { + result.addProperty("status", ResponseStatus.NOT_FOUND.name()); + } else if (payload.getStatus() == TransportProtos.ResponseStatus.FAILURE) { result.addProperty("errorMsg", "Failed to provision device!"); - result.addProperty("status", ProvisionResponseStatus.FAILURE.name()); + result.addProperty("status", ResponseStatus.FAILURE.name()); } else { if (toGateway) { result.addProperty("id", requestId); @@ -445,7 +445,7 @@ public class JsonConverter { break; } result.addProperty("credentialsType", payload.getCredentialsType().name()); - result.addProperty("status", ProvisionResponseStatus.SUCCESS.name()); + result.addProperty("status", ResponseStatus.SUCCESS.name()); } return result; } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 740a2a622f..fa28627882 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -377,7 +377,6 @@ public class DefaultTransportService implements TransportService { AsyncCallbackTemplate.withCallback(response, callback::onSuccess, callback::onError, transportCallbackExecutor); } - @Override public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.SubscriptionInfoProto msg, TransportServiceCallback callback) { if (log.isTraceEnabled()) { @@ -529,6 +528,19 @@ public class DefaultTransportService implements TransportService { } } + @Override + public void process(TransportProtos.SessionInfoProto sessionInfo, TransportProtos.GetFirmwareRequestMsg msg, TransportServiceCallback callback) { + if (checkLimits(sessionInfo, msg, callback)) { + TbProtoQueueMsg protoMsg = + new TbProtoQueueMsg<>(UUID.randomUUID(), TransportProtos.TransportApiRequestMsg.newBuilder().setFirmwareRequestMsg(msg).build()); + + AsyncCallbackTemplate.withCallback(transportApiRequestTemplate.send(protoMsg), response -> { + TransportProtos.GetFirmwareResponseMsg firmwareResponseMsg = response.getValue().getFirmwareResponseMsg(); + callback.onSuccess(firmwareResponseMsg); + }, callback::onError, transportCallbackExecutor); + } + } + @Override public SessionMetaData reportActivity(TransportProtos.SessionInfoProto sessionInfo) { return reportActivityInternal(sessionInfo); @@ -608,11 +620,11 @@ public class DefaultTransportService implements TransportService { sessions.remove(toSessionId(sessionInfo)); } - private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { + private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback) { return checkLimits(sessionInfo, msg, callback, 0); } - private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints) { + private boolean checkLimits(TransportProtos.SessionInfoProto sessionInfo, Object msg, TransportServiceCallback callback, int dataPoints) { if (log.isTraceEnabled()) { log.trace("[{}] Processing msg: {}", toSessionId(sessionInfo), msg); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java index 195f31bd1d..df3291b486 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/firmware/BaseFirmwareService.java @@ -20,6 +20,8 @@ import com.google.common.hash.Hashing; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hibernate.exception.ConstraintViolationException; +import org.springframework.cache.Cache; +import org.springframework.cache.CacheManager; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Firmware; import org.thingsboard.server.common.data.FirmwareInfo; @@ -34,8 +36,10 @@ import org.thingsboard.server.dao.service.PaginatedRemover; import org.thingsboard.server.dao.tenant.TenantDao; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.Optional; +import static org.thingsboard.server.common.data.CacheConstants.FIRMWARE_CACHE; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; @@ -48,11 +52,13 @@ public class BaseFirmwareService implements FirmwareService { private final TenantDao tenantDao; private final FirmwareDao firmwareDao; private final FirmwareInfoDao firmwareInfoDao; + private final CacheManager cacheManager; - public BaseFirmwareService(TenantDao tenantDao, FirmwareDao firmwareDao, FirmwareInfoDao firmwareInfoDao) { + public BaseFirmwareService(TenantDao tenantDao, FirmwareDao firmwareDao, FirmwareInfoDao firmwareInfoDao, CacheManager cacheManager) { this.tenantDao = tenantDao; this.firmwareDao = firmwareDao; this.firmwareInfoDao = firmwareInfoDao; + this.cacheManager = cacheManager; } @Override @@ -60,6 +66,10 @@ public class BaseFirmwareService implements FirmwareService { log.trace("Executing saveFirmwareInfo [{}]", firmwareInfo); firmwareInfoValidator.validate(firmwareInfo, FirmwareInfo::getTenantId); try { + FirmwareId firmwareId = firmwareInfo.getId(); + if (firmwareId != null) { + cacheManager.getCache(FIRMWARE_CACHE).evict(firmwareId.toString()); + } return firmwareInfoDao.save(firmwareInfo.getTenantId(), firmwareInfo); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); @@ -76,6 +86,10 @@ public class BaseFirmwareService implements FirmwareService { log.trace("Executing saveFirmware [{}]", firmware); firmwareValidator.validate(firmware, FirmwareInfo::getTenantId); try { + FirmwareId firmwareId = firmware.getId(); + if (firmwareId != null) { + cacheManager.getCache(FIRMWARE_CACHE).evict(firmwareId.toString()); + } return firmwareDao.save(firmware.getTenantId(), firmware); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); @@ -122,6 +136,8 @@ public class BaseFirmwareService implements FirmwareService { log.trace("Executing deleteFirmware [{}]", firmwareId); validateId(firmwareId, INCORRECT_FIRMWARE_ID + firmwareId); try { + Cache cache = cacheManager.getCache(FIRMWARE_CACHE); + cache.evict(Collections.singletonList(firmwareId)); firmwareDao.removeById(tenantId, firmwareId.getId()); } catch (Exception t) { ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); @@ -278,7 +294,6 @@ public class BaseFirmwareService implements FirmwareService { private PaginatedRemover tenantFirmwareRemover = new PaginatedRemover<>() { - @Override protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { return firmwareInfoDao.findFirmwareInfoByTenantId(id, pageLink);