Browse Source

UI: Merge with master and resolve conflict

pull/9947/head
Artem Dzhereleiko 3 years ago
parent
commit
ddbf2c9552
  1. 2
      application/pom.xml
  2. 4
      application/src/main/data/json/system/widget_bundles/air_quality.json
  3. 2
      application/src/main/data/json/system/widget_bundles/industial_widgets.json
  4. 3
      application/src/main/data/json/system/widget_types/carbon_monoxide__co__chart_card.json
  5. 3
      application/src/main/data/json/system/widget_types/carbon_monoxide__co__chart_card_with_background.json
  6. 3
      application/src/main/data/json/system/widget_types/efficiency_chart_card.json
  7. 3
      application/src/main/data/json/system/widget_types/efficiency_chart_card_with_background.json
  8. 3
      application/src/main/data/json/system/widget_types/flow_rate_chart_card.json
  9. 3
      application/src/main/data/json/system/widget_types/flow_rate_chart_card_with_background.json
  10. 3
      application/src/main/data/json/system/widget_types/fluid_pressure_chart_card.json
  11. 3
      application/src/main/data/json/system/widget_types/fluid_pressure_chart_card_with_background.json
  12. 3
      application/src/main/data/json/system/widget_types/individual_allergy_index__iai__chart_card.json
  13. 3
      application/src/main/data/json/system/widget_types/individual_allergy_index__iai__chart_card_with_background.json
  14. 3
      application/src/main/data/json/system/widget_types/nitrogen_dioxide__no2__chart_card.json
  15. 3
      application/src/main/data/json/system/widget_types/nitrogen_dioxide__no2__chart_card_with_background.json
  16. 5
      application/src/main/data/json/system/widget_types/ozone__o3__chart_card.json
  17. 5
      application/src/main/data/json/system/widget_types/ozone__o3__chart_card_with_background.json
  18. 3
      application/src/main/data/json/system/widget_types/power_consumption_chart_card.json
  19. 3
      application/src/main/data/json/system/widget_types/power_consumption_chart_card_with_background.json
  20. 3
      application/src/main/data/json/system/widget_types/pump_vibration_chart_card.json
  21. 3
      application/src/main/data/json/system/widget_types/pump_vibration_chart_card_with_background.json
  22. 3
      application/src/main/data/json/system/widget_types/rotational_speed_chart_card.json
  23. 3
      application/src/main/data/json/system/widget_types/rotational_speed_chart_card_with_background.json
  24. 3
      application/src/main/data/json/system/widget_types/sulfur_dioxide__so2__chart_card.json
  25. 3
      application/src/main/data/json/system/widget_types/sulfur_dioxide__so2__chart_card_with_background.json
  26. 30
      application/src/main/data/upgrade/3.6.1/schema_update.sql
  27. 23
      application/src/main/data/upgrade/3.6.2/schema_update.sql
  28. 2
      application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java
  29. 19
      application/src/main/java/org/thingsboard/server/controller/BaseController.java
  30. 18
      application/src/main/java/org/thingsboard/server/controller/EdgeController.java
  31. 8
      application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
  32. 103
      application/src/main/java/org/thingsboard/server/controller/ImageController.java
  33. 5
      application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java
  34. 3
      application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java
  35. 39
      application/src/main/java/org/thingsboard/server/service/edge/instructions/DefaultEdgeUpgradeInstructionsService.java
  36. 4
      application/src/main/java/org/thingsboard/server/service/edge/instructions/EdgeUpgradeInstructionsService.java
  37. 2
      application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java
  38. 32
      application/src/main/java/org/thingsboard/server/service/executors/VersionControlExecutor.java
  39. 3
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  40. 2
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java
  41. 16
      application/src/main/java/org/thingsboard/server/service/notification/DefaultNotificationSchedulerService.java
  42. 15
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  43. 66
      application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java
  44. 4
      application/src/main/java/org/thingsboard/server/service/resource/TbImageService.java
  45. 3
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/NotificationTargetExportService.java
  46. 8
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/ResourceExportService.java
  47. 7
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java
  48. 12
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java
  49. 46
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  50. 172
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java
  51. 5
      application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java
  52. 10
      application/src/main/resources/thingsboard.yml
  53. 178
      application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java
  54. 42
      application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java
  55. 80
      application/src/test/java/org/thingsboard/server/controller/ImageControllerTest.java
  56. 2
      common/actor/pom.xml
  57. 2
      common/cache/pom.xml
  58. 2
      common/cluster-api/pom.xml
  59. 2
      common/coap-server/pom.xml
  60. 2
      common/dao-api/pom.xml
  61. 2
      common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java
  62. 1
      common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java
  63. 2
      common/data/pom.xml
  64. 1
      common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java
  65. 18
      common/data/src/main/java/org/thingsboard/server/common/data/ImageExportData.java
  66. 1
      common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java
  67. 15
      common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java
  68. 23
      common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java
  69. 2
      common/edge-api/pom.xml
  70. 2
      common/message/pom.xml
  71. 2
      common/pom.xml
  72. 2
      common/proto/pom.xml
  73. 7
      common/proto/src/main/proto/queue.proto
  74. 2
      common/queue/pom.xml
  75. 12
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java
  76. 42
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java
  77. 3
      common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java
  78. 2
      common/script/pom.xml
  79. 2
      common/script/remote-js-client/pom.xml
  80. 2
      common/script/script-api/pom.xml
  81. 21
      common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java
  82. 2
      common/stats/pom.xml
  83. 2
      common/transport/coap/pom.xml
  84. 2
      common/transport/http/pom.xml
  85. 2
      common/transport/lwm2m/pom.xml
  86. 2
      common/transport/mqtt/pom.xml
  87. 2
      common/transport/pom.xml
  88. 2
      common/transport/snmp/pom.xml
  89. 2
      common/transport/transport-api/pom.xml
  90. 2
      common/util/pom.xml
  91. 2
      common/version-control/pom.xml
  92. 15
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java
  93. 9
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
  94. 2
      dao/pom.xml
  95. 13
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java
  96. 2
      dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java
  97. 12
      dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java
  98. 12
      dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java
  99. 28
      dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java
  100. 36
      dao/src/main/java/org/thingsboard/server/dao/resource/ImageCacheKey.java

2
application/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>thingsboard</artifactId>
</parent>
<artifactId>application</artifactId>

4
application/src/main/data/json/system/widget_bundles/air_quality.json

File diff suppressed because one or more lines are too long

2
application/src/main/data/json/system/widget_bundles/industial_widgets.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/carbon_monoxide__co__chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/carbon_monoxide__co__chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/efficiency_chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/efficiency_chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/flow_rate_chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/flow_rate_chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/fluid_pressure_chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/fluid_pressure_chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/individual_allergy_index__iai__chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/individual_allergy_index__iai__chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/nitrogen_dioxide__no2__chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/nitrogen_dioxide__no2__chart_card_with_background.json

File diff suppressed because one or more lines are too long

5
application/src/main/data/json/system/widget_types/ozone__o3__chart_card.json

File diff suppressed because one or more lines are too long

5
application/src/main/data/json/system/widget_types/ozone__o3__chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/power_consumption_chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/power_consumption_chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/pump_vibration_chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/pump_vibration_chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/rotational_speed_chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/rotational_speed_chart_card_with_background.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/sulfur_dioxide__so2__chart_card.json

File diff suppressed because one or more lines are too long

3
application/src/main/data/json/system/widget_types/sulfur_dioxide__so2__chart_card_with_background.json

File diff suppressed because one or more lines are too long

30
application/src/main/data/upgrade/3.6.1/schema_update.sql

@ -16,6 +16,28 @@
-- RESOURCES UPDATE START
ALTER TABLE resource ADD COLUMN IF NOT EXISTS descriptor varchar;
ALTER TABLE resource ADD COLUMN IF NOT EXISTS preview bytea;
ALTER TABLE resource ADD COLUMN IF NOT EXISTS external_id uuid;
ALTER TABLE resource ADD COLUMN IF NOT EXISTS is_public boolean default true;
ALTER TABLE resource ADD COLUMN IF NOT EXISTS public_resource_key varchar(32) unique;
CREATE INDEX IF NOT EXISTS idx_resource_etag ON resource(tenant_id, etag);
CREATE INDEX IF NOT EXISTS idx_resource_type_public_resource_key ON resource(resource_type, public_resource_key);
CREATE OR REPLACE FUNCTION generate_resource_public_key()
RETURNS text AS $$
DECLARE
chars text := 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
result text := '';
BEGIN
FOR i IN 1..32 LOOP
result := result || substr(chars, floor(random()*62)::int + 1, 1);
END LOOP;
RETURN result;
END;
$$ LANGUAGE plpgsql;
DO
$$
BEGIN
@ -24,15 +46,13 @@ $$
ALTER TABLE resource ADD COLUMN data bytea;
UPDATE resource SET data = decode(base64_data, 'base64') WHERE base64_data IS NOT NULL;
ALTER TABLE resource DROP COLUMN base64_data;
ELSE
UPDATE resource SET public_resource_key = generate_resource_public_key() WHERE resource_type = 'IMAGE' AND public_resource_key IS NULL;
END IF;
END;
$$;
ALTER TABLE resource ADD COLUMN IF NOT EXISTS descriptor varchar;
ALTER TABLE resource ADD COLUMN IF NOT EXISTS preview bytea;
ALTER TABLE resource ADD COLUMN IF NOT EXISTS external_id uuid;
CREATE INDEX IF NOT EXISTS idx_resource_etag ON resource(tenant_id, etag);
DROP FUNCTION generate_resource_public_key;
-- RESOURCES UPDATE END

23
application/src/main/data/upgrade/3.6.2/schema_update.sql

@ -0,0 +1,23 @@
--
-- Copyright © 2016-2023 The Thingsboard Authors
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.
--
-- RULE NODE INDEXES UPDATE START
DROP INDEX IF EXISTS idx_rule_node_type;
DROP INDEX IF EXISTS idx_rule_node_type_configuration_version;
CREATE INDEX IF NOT EXISTS idx_rule_node_type_id_configuration_version ON rule_node(type, id, configuration_version);
-- RULE NODE INDEXES UPDATE END

2
application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java

@ -75,7 +75,7 @@ public class ThingsboardSecurityConfiguration {
public static final String FORM_BASED_LOGIN_ENTRY_POINT = "/api/auth/login";
public static final String PUBLIC_LOGIN_ENTRY_POINT = "/api/auth/login/public";
public static final String TOKEN_REFRESH_ENTRY_POINT = "/api/auth/token";
protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[] {"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**"};
protected static final String[] NON_TOKEN_BASED_AUTH_ENTRY_POINTS = new String[] {"/index.html", "/assets/**", "/static/**", "/api/noauth/**", "/webjars/**", "/api/license/**", "/api/images/public/**"};
public static final String TOKEN_BASED_AUTH_ENTRY_POINT = "/api/**";
public static final String WS_ENTRY_POINT = "/api/ws/**";
public static final String MAIL_OAUTH2_PROCESSING_ENTRY_POINT = "/api/admin/mail/oauth2/code";

19
application/src/main/java/org/thingsboard/server/controller/BaseController.java

@ -31,6 +31,7 @@ import org.springframework.web.bind.MethodArgumentNotValidException;
import org.springframework.web.bind.annotation.ExceptionHandler;
import org.springframework.web.context.request.async.AsyncRequestTimeoutException;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.Dashboard;
@ -833,18 +834,14 @@ public abstract class BaseController {
}
protected <T> DeferredResult<T> wrapFuture(ListenableFuture<T> future) {
final DeferredResult<T> deferredResult = new DeferredResult<>();
Futures.addCallback(future, new FutureCallback<>() {
@Override
public void onSuccess(T result) {
deferredResult.setResult(result);
}
DeferredResult<T> deferredResult = new DeferredResult<>(); // Timeout of spring.mvc.async.request-timeout is used
DonAsynchron.withCallback(future, deferredResult::setResult, deferredResult::setErrorResult);
return deferredResult;
}
@Override
public void onFailure(Throwable t) {
deferredResult.setErrorResult(t);
}
}, MoreExecutors.directExecutor());
protected <T> DeferredResult<T> wrapFuture(ListenableFuture<T> future, long timeoutMs) {
DeferredResult<T> deferredResult = new DeferredResult<>(timeoutMs);
DonAsynchron.withCallback(future, deferredResult::setResult, deferredResult::setErrorResult);
return deferredResult;
}

18
application/src/main/java/org/thingsboard/server/controller/EdgeController.java

@ -594,4 +594,22 @@ public class EdgeController extends BaseController {
throw new ThingsboardException("Edges support disabled", ThingsboardErrorCode.GENERAL);
}
}
@ApiOperation(value = "Is edge upgrade enabled (isEdgeUpgradeAvailable)",
notes = "Returns 'true' if upgrade available for connected edge, 'false' - otherwise.")
@PreAuthorize("hasAnyAuthority('TENANT_ADMIN')")
@RequestMapping(value = "/edge/{edgeId}/upgrade/available", method = RequestMethod.GET)
@ResponseBody
public boolean isEdgeUpgradeAvailable(
@ApiParam(value = EDGE_ID_PARAM_DESCRIPTION, required = true)
@PathVariable("edgeId") String strEdgeId) throws Exception {
if (isEdgesEnabled() && edgeUpgradeServiceOpt.isPresent()) {
EdgeId edgeId = new EdgeId(toUUID(strEdgeId));
edgeId = checkNotNull(edgeId);
Edge edge = checkEdgeId(edgeId, Operation.READ);
return edgeUpgradeServiceOpt.get().isUpgradeAvailable(edge.getTenantId(), edge.getId());
} else {
throw new ThingsboardException("Edges support disabled", ThingsboardErrorCode.GENERAL);
}
}
}

8
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@ -21,6 +21,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.RequiredArgsConstructor;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
@ -83,6 +84,8 @@ public class EntitiesVersionControlController extends BaseController {
private final EntitiesVersionControlService versionControlService;
@Value("${queue.vc.request-timeout:180000}")
private int vcRequestTimeout;
@ApiOperation(value = "Save entities version (saveEntitiesVersion)", notes = "" +
"Creates a new version of entities (or a single entity) by request.\n" +
@ -515,4 +518,9 @@ public class EntitiesVersionControlController extends BaseController {
}, MoreExecutors.directExecutor()));
}
@Override
protected <T> DeferredResult<T> wrapFuture(ListenableFuture<T> future) {
return wrapFuture(future, vcRequestTimeout);
}
}

103
application/src/main/java/org/thingsboard/server/controller/ImageController.java

@ -15,7 +15,6 @@
*/
package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.JsonProcessingException;
import io.swagger.annotations.ApiParam;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ -50,6 +49,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.security.Authority;
import org.thingsboard.server.common.data.util.ThrowingSupplier;
import org.thingsboard.server.dao.resource.ImageCacheKey;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.service.validator.ResourceDataValidator;
@ -89,6 +89,10 @@ public class ImageController extends BaseController {
private static final String SYSTEM_IMAGE = "system";
private static final String TENANT_IMAGE = "tenant";
private static final String IMAGE_TYPE_PARAM_DESCRIPTION = "Type of the image: tenant or system";
private static final String IMAGE_TYPE_PARAM_ALLOWABLE_VALUES = "tenant, system";
private static final String IMAGE_KEY_PARAM_DESCRIPTION = "Image resource key, for example thermostats_dashboard_background.jpeg";
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@PostMapping("/api/image")
public TbResourceInfo uploadImage(@RequestPart MultipartFile file,
@ -110,12 +114,15 @@ public class ImageController extends BaseController {
descriptor.setMediaType(file.getContentType());
image.setDescriptorValue(descriptor);
image.setData(file.getBytes());
image.setPublic(true);
return tbImageService.save(image, user);
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@PutMapping(IMAGE_URL)
public TbResourceInfo updateImage(@PathVariable String type,
public TbResourceInfo updateImage(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key,
@RequestPart MultipartFile file) throws Exception {
TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.WRITE);
@ -133,29 +140,65 @@ public class ImageController extends BaseController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@PutMapping(IMAGE_URL + "/info")
public TbResourceInfo updateImageInfo(@PathVariable String type,
public TbResourceInfo updateImageInfo(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key,
@RequestBody TbResourceInfo newImageInfo) throws ThingsboardException {
@RequestBody TbResourceInfo request) throws ThingsboardException {
TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.WRITE);
TbResourceInfo newImageInfo = new TbResourceInfo(imageInfo);
newImageInfo.setTitle(request.getTitle());
return tbImageService.save(newImageInfo, imageInfo, getCurrentUser());
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@PutMapping(IMAGE_URL + "/public/{isPublic}")
public TbResourceInfo updateImagePublicStatus(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key,
@PathVariable boolean isPublic) throws ThingsboardException {
TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.WRITE);
imageInfo.setTitle(newImageInfo.getTitle());
return tbImageService.save(imageInfo, getCurrentUser());
TbResourceInfo newImageInfo = new TbResourceInfo(imageInfo);
newImageInfo.setPublic(isPublic);
return tbImageService.save(newImageInfo, imageInfo, getCurrentUser());
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@GetMapping(value = IMAGE_URL, produces = "image/*")
public ResponseEntity<ByteArrayResource> downloadImage(@PathVariable String type,
public ResponseEntity<ByteArrayResource> downloadImage(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws Exception {
return downloadIfChanged(type, key, etag, false);
}
@GetMapping(value = "/api/images/public/{publicResourceKey}", produces = "image/*")
public ResponseEntity<ByteArrayResource> downloadPublicImage(@PathVariable String publicResourceKey,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws Exception {
ImageCacheKey cacheKey = ImageCacheKey.forPublicImage(publicResourceKey);
return downloadIfChanged(cacheKey, etag, () -> imageService.getPublicImageInfoByKey(publicResourceKey));
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@GetMapping(value = IMAGE_URL + "/export")
public ImageExportData exportImage(@PathVariable String type, @PathVariable String key) throws Exception {
public ImageExportData exportImage(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key) throws Exception {
TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.READ);
ImageDescriptor descriptor = imageInfo.getDescriptor(ImageDescriptor.class);
byte[] data = imageService.getImageData(imageInfo.getTenantId(), imageInfo.getId());
return new ImageExportData(descriptor.getMediaType(), imageInfo.getFileName(), imageInfo.getTitle(), imageInfo.getResourceKey(), Base64Utils.encodeToString(data));
return ImageExportData.builder()
.mediaType(descriptor.getMediaType())
.fileName(imageInfo.getFileName())
.title(imageInfo.getTitle())
.resourceKey(imageInfo.getResourceKey())
.isPublic(imageInfo.isPublic())
.publicResourceKey(imageInfo.getPublicResourceKey())
.data(Base64Utils.encodeToString(data))
.build();
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@ -172,19 +215,22 @@ public class ImageController extends BaseController {
} else {
image.setTitle(imageData.getFileName());
}
image.setResourceKey(imageData.getResourceKey());
image.setResourceType(ResourceType.IMAGE);
image.setResourceKey(imageData.getResourceKey());
image.setPublic(imageData.isPublic());
image.setPublicResourceKey(imageData.getPublicResourceKey());
ImageDescriptor descriptor = new ImageDescriptor();
descriptor.setMediaType(imageData.getMediaType());
image.setDescriptorValue(descriptor);
image.setData(Base64Utils.decodeFromString(imageData.getData()));
return tbImageService.save(image, user);
}
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')")
@GetMapping(value = IMAGE_URL + "/preview", produces = "image/png")
public ResponseEntity<ByteArrayResource> downloadImagePreview(@PathVariable String type,
public ResponseEntity<ByteArrayResource> downloadImagePreview(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key,
@RequestHeader(name = HttpHeaders.IF_NONE_MATCH, required = false) String etag) throws Exception {
return downloadIfChanged(type, key, etag, true);
@ -192,7 +238,9 @@ public class ImageController extends BaseController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@GetMapping(IMAGE_URL + "/info")
public TbResourceInfo getImageInfo(@PathVariable String type,
public TbResourceInfo getImageInfo(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key) throws ThingsboardException {
return checkImageInfo(type, key, Operation.READ);
}
@ -223,7 +271,9 @@ public class ImageController extends BaseController {
@PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')")
@DeleteMapping(IMAGE_URL)
public ResponseEntity<TbImageDeleteResult> deleteImage(@PathVariable String type,
public ResponseEntity<TbImageDeleteResult> deleteImage(@ApiParam(value = IMAGE_TYPE_PARAM_DESCRIPTION, allowableValues = IMAGE_TYPE_PARAM_ALLOWABLE_VALUES, required = true)
@PathVariable String type,
@ApiParam(value = IMAGE_KEY_PARAM_DESCRIPTION, required = true)
@PathVariable String key,
@RequestParam(name = "force", required = false) boolean force) throws ThingsboardException {
TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.DELETE);
@ -231,32 +281,39 @@ public class ImageController extends BaseController {
return (result.isSuccess() ? ResponseEntity.ok() : ResponseEntity.badRequest()).body(result);
}
private ResponseEntity<ByteArrayResource> downloadIfChanged(String type, String key, String etag, boolean preview) throws ThingsboardException, JsonProcessingException {
ImageCacheKey cacheKey = new ImageCacheKey(getTenantId(type), key, preview);
private ResponseEntity<ByteArrayResource> downloadIfChanged(String type, String key, String etag, boolean preview) throws Exception {
ImageCacheKey cacheKey = ImageCacheKey.forImage(getTenantId(type), key, preview);
return downloadIfChanged(cacheKey, etag, () -> checkImageInfo(type, key, Operation.READ));
}
private ResponseEntity<ByteArrayResource> downloadIfChanged(ImageCacheKey cacheKey, String etag, ThrowingSupplier<TbResourceInfo> imageInfoSupplier) throws Exception {
if (StringUtils.isNotEmpty(etag)) {
etag = StringUtils.remove(etag, '\"'); // etag is wrapped in double quotes due to HTTP specification
if (etag.equals(tbImageService.getETag(cacheKey))) {
return ResponseEntity.status(HttpStatus.NOT_MODIFIED).build();
}
}
TenantId tenantId = getTenantId();
TbResourceInfo imageInfo = checkImageInfo(type, key, Operation.READ);
TbResourceInfo imageInfo = checkNotNull(imageInfoSupplier.get());
String fileName = imageInfo.getFileName();
ImageDescriptor descriptor = imageInfo.getDescriptor(ImageDescriptor.class);
byte[] data;
if (preview) {
if (cacheKey.isPreview()) {
descriptor = descriptor.getPreviewDescriptor();
data = imageService.getImagePreview(tenantId, imageInfo.getId());
data = imageService.getImagePreview(imageInfo.getTenantId(), imageInfo.getId());
} else {
data = imageService.getImageData(tenantId, imageInfo.getId());
data = imageService.getImageData(imageInfo.getTenantId(), imageInfo.getId());
}
tbImageService.putETag(cacheKey, descriptor.getEtag());
var result = ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + fileName)
.header("x-filename", fileName)
.header("Content-Type", descriptor.getMediaType())
.contentLength(data.length)
.eTag(descriptor.getEtag());
if (!cacheKey.isPublic()) {
result
.header(HttpHeaders.CONTENT_DISPOSITION, "attachment;filename=" + fileName)
.header("x-filename", fileName);
}
if (systemImagesBrowserTtlInMinutes > 0 && imageInfo.getTenantId().isSysTenantId()) {
result.cacheControl(CacheControl.maxAge(systemImagesBrowserTtlInMinutes, TimeUnit.MINUTES));
} else if (tenantImagesBrowserTtlInMinutes > 0 && !imageInfo.getTenantId().isSysTenantId()) {

5
application/src/main/java/org/thingsboard/server/controller/SystemInfoController.java

@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.settings.UserSettings;
import org.thingsboard.server.common.data.settings.UserSettingsType;
import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.security.model.UserPrincipal;
@ -130,6 +131,10 @@ public class SystemInfoController extends BaseController {
}
systemParams.setUserSettings(userSettingsNode);
systemParams.setMaxDatapointsLimit(maxDatapointsLimit);
if (!currentUser.isSystemAdmin()) {
DefaultTenantProfileConfiguration tenantProfileConfiguration = tenantProfileCache.get(tenantId).getDefaultProfileConfiguration();
systemParams.setMaxResourceSize(tenantProfileConfiguration.getMaxResourceSize());
}
return systemParams;
}

3
application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java

@ -276,6 +276,9 @@ public class ThingsboardInstallService {
} else {
log.info("Skipping images migration. Run the upgrade with fromVersion as '3.6.2-images' to migrate");
}
case "3.6.2":
log.info("Upgrading ThingsBoard from version 3.6.2 to 3.6.3 ...");
databaseEntitiesUpgradeService.upgradeDatabase("3.6.2");
//TODO DON'T FORGET to update switch statement in the CacheCleanupService if you need to clear the cache
break;
default:

39
application/src/main/java/org/thingsboard/server/service/edge/instructions/DefaultEdgeUpgradeInstructionsService.java

@ -21,8 +21,13 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EdgeUpgradeInfo;
import org.thingsboard.server.common.data.edge.EdgeInstructions;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.install.InstallScripts;
@ -32,6 +37,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
@Service
@Slf4j
@ -47,6 +53,7 @@ public class DefaultEdgeUpgradeInstructionsService implements EdgeUpgradeInstruc
private static final String UPGRADE_DIR = "upgrade";
private final InstallScripts installScripts;
private final AttributesService attributesService;
@Value("${app.version:unknown}")
@Setter
@ -74,13 +81,41 @@ public class DefaultEdgeUpgradeInstructionsService implements EdgeUpgradeInstruc
}
}
@Override
public boolean isUpgradeAvailable(TenantId tenantId, EdgeId edgeId) throws Exception {
Optional<AttributeKvEntry> attributeKvEntryOpt = attributesService.find(tenantId, edgeId, DataConstants.SERVER_SCOPE, DataConstants.EDGE_VERSION_ATTR_KEY).get();
if (attributeKvEntryOpt.isPresent()) {
String edgeVersionFormatted = convertEdgeVersionToDocsFormat(attributeKvEntryOpt.get().getValueAsString());
return isVersionGreaterOrEqualsThan(edgeVersionFormatted, "3.6.0") && !isVersionGreaterOrEqualsThan(edgeVersionFormatted, appVersion);
}
return false;
}
private boolean isVersionGreaterOrEqualsThan(String version1, String version2) {
String[] v1 = version1.split("\\.");
String[] v2 = version2.split("\\.");
int length = Math.max(v1.length, v2.length);
for (int i = 0; i < length; i++) {
int num1 = i < v1.length ? Integer.parseInt(v1[i]) : 0;
int num2 = i < v2.length ? Integer.parseInt(v2[i]) : 0;
if (num1 < num2) {
return false;
} else if (num1 > num2) {
return true;
}
}
return true;
}
private EdgeInstructions getDockerUpgradeInstructions(String tbVersion, String currentEdgeVersion) {
EdgeUpgradeInfo edgeUpgradeInfo = upgradeVersionHashMap.get(currentEdgeVersion);
if (edgeUpgradeInfo == null || edgeUpgradeInfo.getNextEdgeVersion() == null || tbVersion.equals(currentEdgeVersion)) {
return new EdgeInstructions("Edge upgrade instruction for " + currentEdgeVersion + "EDGE is not available.");
}
StringBuilder result = new StringBuilder(readFile(resolveFile("docker", "upgrade_preparing.md")));
while (edgeUpgradeInfo.getNextEdgeVersion() != null || !tbVersion.equals(currentEdgeVersion)) {
while (edgeUpgradeInfo.getNextEdgeVersion() != null && !tbVersion.equals(currentEdgeVersion)) {
String edgeVersion = edgeUpgradeInfo.getNextEdgeVersion();
String dockerUpgradeInstructions = readFile(resolveFile("docker", "instructions.md"));
if (edgeUpgradeInfo.isRequiresUpdateDb()) {
@ -109,7 +144,7 @@ public class DefaultEdgeUpgradeInstructionsService implements EdgeUpgradeInstruc
String upgrade_preparing = readFile(resolveFile("upgrade_preparing.md"));
upgrade_preparing = upgrade_preparing.replace("${OS}", os.equals("centos") ? "RHEL/CentOS 7/8" : "Ubuntu");
StringBuilder result = new StringBuilder(upgrade_preparing);
while (edgeUpgradeInfo.getNextEdgeVersion() != null || !tbVersion.equals(currentEdgeVersion)) {
while (edgeUpgradeInfo.getNextEdgeVersion() != null && !tbVersion.equals(currentEdgeVersion)) {
String edgeVersion = edgeUpgradeInfo.getNextEdgeVersion();
String linuxUpgradeInstructions = readFile(resolveFile(os, "instructions.md"));
if (edgeUpgradeInfo.isRequiresUpdateDb()) {

4
application/src/main/java/org/thingsboard/server/service/edge/instructions/EdgeUpgradeInstructionsService.java

@ -17,6 +17,8 @@ package org.thingsboard.server.service.edge.instructions;
import org.thingsboard.server.common.data.EdgeUpgradeInfo;
import org.thingsboard.server.common.data.edge.EdgeInstructions;
import org.thingsboard.server.common.data.id.EdgeId;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.Map;
@ -27,4 +29,6 @@ public interface EdgeUpgradeInstructionsService {
void updateInstructionMap(Map<String, EdgeUpgradeInfo> upgradeVersions);
void setAppVersion(String version);
boolean isUpgradeAvailable(TenantId tenantId, EdgeId edgeId) throws Exception;
}

2
application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java

@ -822,7 +822,7 @@ public final class EdgeGrpcSession implements Closeable {
}
private void processSaveEdgeVersionAsAttribute(String edgeVersion) {
AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(new StringDataEntry("edgeVersion", edgeVersion), System.currentTimeMillis());
AttributeKvEntry attributeKvEntry = new BaseAttributeKvEntry(new StringDataEntry(DataConstants.EDGE_VERSION_ATTR_KEY, edgeVersion), System.currentTimeMillis());
ctx.getAttributesService().save(this.tenantId, this.edge.getId(), DataConstants.SERVER_SCOPE, attributeKvEntry);
}

32
application/src/main/java/org/thingsboard/server/service/executors/VersionControlExecutor.java

@ -0,0 +1,32 @@
/**
* Copyright © 2016-2023 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.service.executors;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.thingsboard.common.util.AbstractListeningExecutor;
@Component
public class VersionControlExecutor extends AbstractListeningExecutor {
@Value("${vc.thread_pool_size:6}")
private int threadPoolSize;
@Override
protected int getThreadPollSize() {
return threadPoolSize;
}
}

3
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -772,6 +772,9 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
}
});
break;
case "3.6.2":
updateSchema("3.6.2", 3006002, "3.6.3", 3006003, null);
break;
default:
throw new RuntimeException("Unable to upgrade SQL database, unsupported fromVersion: " + fromVersion);
}

2
application/src/main/java/org/thingsboard/server/service/install/update/DefaultCacheCleanupService.java

@ -27,6 +27,7 @@ import org.springframework.stereotype.Service;
import java.util.Objects;
import java.util.Optional;
import static org.thingsboard.server.common.data.CacheConstants.RESOURCE_INFO_CACHE;
import static org.thingsboard.server.common.data.CacheConstants.SECURITY_SETTINGS_CACHE;
@RequiredArgsConstructor
@ -91,6 +92,7 @@ public class DefaultCacheCleanupService implements CacheCleanupService {
case "3.6.1":
log.info("Clearing cache to upgrade from version 3.6.1 to 3.6.2");
clearCacheByName(SECURITY_SETTINGS_CACHE);
clearCacheByName(RESOURCE_INFO_CACHE);
break;
default:
//Do nothing, since cache cleanup is optional.

16
application/src/main/java/org/thingsboard/server/service/notification/DefaultNotificationSchedulerService.java

@ -21,6 +21,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.rule.engine.api.NotificationCenter;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.NotificationRequestId;
@ -35,12 +36,12 @@ import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.notification.NotificationRequestService;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.executors.NotificationExecutorService;
import org.thingsboard.server.service.partition.AbstractPartitionBasedService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
@ -48,6 +49,8 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
@ -55,16 +58,16 @@ import java.util.concurrent.TimeUnit;
@Service
@RequiredArgsConstructor
@Slf4j
@SuppressWarnings("UnstableApiUsage")
public class DefaultNotificationSchedulerService extends AbstractPartitionBasedService<NotificationRequestId> implements NotificationSchedulerService {
private final NotificationCenter notificationCenter;
private final NotificationRequestService notificationRequestService;
private final SchedulerComponent scheduler;
private final NotificationExecutorService notificationExecutor;
private final ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(ThingsBoardThreadFactory.forName("notification-scheduler"));
private final Map<NotificationRequestId, ScheduledRequestMetadata> scheduledNotificationRequests = new ConcurrentHashMap<>();
@Override
@PostConstruct
public void init() {
super.init();
@ -166,6 +169,13 @@ public class DefaultNotificationSchedulerService extends AbstractPartitionBasedS
return "notifications-scheduler";
}
@Override
@PreDestroy
public void stop() {
super.stop();
scheduler.shutdownNow();
}
@Data
private static class ScheduledRequestMetadata {
private final TenantId tenantId;

15
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -44,8 +44,10 @@ import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg;
import org.thingsboard.server.common.stats.StatsFactory;
import org.thingsboard.server.common.util.ProtoUtils;
import org.thingsboard.server.dao.resource.ImageCacheKey;
import org.thingsboard.server.dao.tenant.TbTenantProfileCache;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.DeviceStateServiceMsgProto;
@ -59,6 +61,7 @@ import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAlarmUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbAttributeUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbEntitySubEventProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesDeleteProto;
import org.thingsboard.server.gen.transport.TransportProtos.TbTimeSeriesUpdateProto;
import org.thingsboard.server.gen.transport.TransportProtos.ToCoreMsg;
@ -66,7 +69,6 @@ import org.thingsboard.server.gen.transport.TransportProtos.ToCoreNotificationMs
import org.thingsboard.server.gen.transport.TransportProtos.ToOtaPackageStateServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ToUsageStatsServiceMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TransportToDeviceActorMsg;
import org.thingsboard.server.gen.transport.TransportProtos.TbEntitySubEventProto;
import org.thingsboard.server.queue.TbQueueConsumer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.discovery.PartitionService;
@ -83,10 +85,8 @@ import org.thingsboard.server.service.profile.TbAssetProfileCache;
import org.thingsboard.server.service.profile.TbDeviceProfileCache;
import org.thingsboard.server.service.queue.processing.AbstractConsumerService;
import org.thingsboard.server.service.queue.processing.IdMsgPair;
import org.thingsboard.server.dao.resource.ImageCacheKey;
import org.thingsboard.server.service.resource.TbImageService;
import org.thingsboard.server.service.rpc.TbCoreDeviceRpcService;
import org.thingsboard.server.common.msg.rpc.ToDeviceRpcRequestActorMsg;
import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService;
import org.thingsboard.server.service.state.DeviceStateService;
import org.thingsboard.server.service.subscription.SubscriptionManagerService;
@ -560,8 +560,13 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
private void forwardToResourceService(TransportProtos.ResourceCacheInvalidateMsg msg, TbCallback callback) {
var tenantId = new TenantId(new UUID(msg.getTenantIdMSB(), msg.getTenantIdLSB()));
imageService.evictETag(new ImageCacheKey(tenantId, msg.getResourceKey(), false));
imageService.evictETag(new ImageCacheKey(tenantId, msg.getResourceKey(), true));
msg.getKeysList().stream().map(cacheKeyProto -> {
if (cacheKeyProto.hasResourceKey()) {
return ImageCacheKey.forImage(tenantId, cacheKeyProto.getResourceKey());
} else {
return ImageCacheKey.forPublicImage(cacheKeyProto.getPublicResourceKey());
}
}).forEach(imageService::evictETags);
callback.onSuccess();
}

66
application/src/main/java/org/thingsboard/server/service/resource/DefaultTbImageService.java

@ -38,8 +38,11 @@ import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
@Slf4j
@ -72,8 +75,11 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb
}
@Override
public void evictETag(ImageCacheKey imageCacheKey) {
public void evictETags(ImageCacheKey imageCacheKey) {
etagCache.invalidate(imageCacheKey);
if (imageCacheKey.getPublicResourceKey() == null) {
etagCache.invalidate(imageCacheKey.withPreview(true));
}
}
@Override
@ -82,28 +88,32 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb
TenantId tenantId = image.getTenantId();
try {
var oldEtag = getEtag(image);
TbResourceInfo existingImage = null;
if (image.getId() == null && StringUtils.isNotEmpty(image.getResourceKey())) {
var existingImage = imageService.getImageInfoByTenantIdAndKey(tenantId, image.getResourceKey());
existingImage = imageService.getImageInfoByTenantIdAndKey(tenantId, image.getResourceKey());
if (existingImage != null) {
image.setId(existingImage.getId());
}
}
TbResourceInfo savedImage = imageService.saveImage(image);
notificationEntityService.logEntityAction(tenantId, savedImage.getId(), savedImage, actionType, user);
List<ImageCacheKey> toEvict = new ArrayList<>();
if (oldEtag.isPresent()) {
var newEtag = getEtag(savedImage);
if (newEtag.isPresent() && !oldEtag.get().equals(newEtag.get())) {
evictETag(new ImageCacheKey(image.getTenantId(), image.getResourceKey(), false));
evictETag(new ImageCacheKey(image.getTenantId(), image.getResourceKey(), true));
clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder()
.setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setResourceKey(image.getResourceKey())
.build())
.build());
toEvict.add(ImageCacheKey.forImage(tenantId, image.getResourceKey()));
if (image.isPublic()) {
toEvict.add(ImageCacheKey.forPublicImage(savedImage.getPublicResourceKey()));
}
}
}
if (existingImage != null && image.isPublic() != existingImage.isPublic()) {
toEvict.add(ImageCacheKey.forPublicImage(image.getPublicResourceKey()));
}
if (!toEvict.isEmpty()) {
evictFromCache(tenantId, toEvict);
}
return savedImage;
} catch (Exception e) {
image.setData(null);
@ -124,12 +134,16 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb
}
@Override
public TbResourceInfo save(TbResourceInfo imageInfo, User user) {
public TbResourceInfo save(TbResourceInfo imageInfo, TbResourceInfo oldImageInfo, User user) {
TenantId tenantId = imageInfo.getTenantId();
TbResourceId imageId = imageInfo.getId();
try {
imageInfo = imageService.saveImageInfo(imageInfo);
notificationEntityService.logEntityAction(tenantId, imageId, imageInfo, ActionType.UPDATED, user);
if (imageInfo.isPublic() != oldImageInfo.isPublic()) {
evictFromCache(tenantId, List.of(ImageCacheKey.forPublicImage(imageInfo.getPublicResourceKey())));
}
return imageInfo;
} catch (Exception e) {
notificationEntityService.logEntityAction(tenantId, imageId, imageInfo, ActionType.UPDATED, user, e);
@ -145,15 +159,13 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb
TbImageDeleteResult result = imageService.deleteImage(imageInfo, force);
if (result.isSuccess()) {
notificationEntityService.logEntityAction(tenantId, imageId, imageInfo, ActionType.DELETED, user, imageId.toString());
evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), false));
evictETag(new ImageCacheKey(tenantId, imageInfo.getResourceKey(), true));
clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder()
.setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setResourceKey(imageInfo.getResourceKey())
.build())
.build());
List<ImageCacheKey> toEvict = new ArrayList<>();
toEvict.add(ImageCacheKey.forImage(tenantId, imageInfo.getResourceKey()));
if (imageInfo.isPublic()) {
toEvict.add(ImageCacheKey.forPublicImage(imageInfo.getPublicResourceKey()));
}
evictFromCache(tenantId, toEvict);
}
return result;
} catch (Exception e) {
@ -161,4 +173,16 @@ public class DefaultTbImageService extends AbstractTbEntityService implements Tb
throw e;
}
}
private void evictFromCache(TenantId tenantId, List<ImageCacheKey> toEvict) {
toEvict.forEach(this::evictETags);
clusterService.broadcastToCore(TransportProtos.ToCoreNotificationMsg.newBuilder()
.setResourceCacheInvalidateMsg(TransportProtos.ResourceCacheInvalidateMsg.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.addAllKeys(toEvict.stream().map(ImageCacheKey::toProto).collect(Collectors.toList()))
.build())
.build());
}
}

4
application/src/main/java/org/thingsboard/server/service/resource/TbImageService.java

@ -25,7 +25,7 @@ public interface TbImageService {
TbResourceInfo save(TbResource image, User user) throws Exception;
TbResourceInfo save(TbResourceInfo imageInfo, User user);
TbResourceInfo save(TbResourceInfo imageInfo, TbResourceInfo oldImageInfo, User user);
TbImageDeleteResult delete(TbResourceInfo imageInfo, User user, boolean force);
@ -33,6 +33,6 @@ public interface TbImageService {
void putETag(ImageCacheKey imageCacheKey, String etag);
void evictETag(ImageCacheKey imageCacheKey);
void evictETags(ImageCacheKey imageCacheKey);
}

3
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/NotificationTargetExportService.java

@ -43,6 +43,9 @@ public class NotificationTargetExportService extends BaseEntityExportService<Not
CustomerUsersFilter customerUsersFilter = (CustomerUsersFilter) usersFilter;
customerUsersFilter.setCustomerId(getExternalIdOrElseInternal(ctx, new CustomerId(customerUsersFilter.getCustomerId())).getId());
break;
case USER_LIST:
// users list stays as is and is replaced with current user id on import (due to user entities not being supported by VC)
break;
}
}
}

8
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/ResourceExportService.java

@ -18,9 +18,11 @@ package org.thingsboard.server.service.sync.ie.exporting.impl;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import java.util.Set;
@ -28,6 +30,12 @@ import java.util.Set;
@TbCoreComponent
public class ResourceExportService extends BaseEntityExportService<TbResourceId, TbResource, EntityExportData<TbResource>> {
@Override
protected void setAdditionalExportData(EntitiesExportCtx<?> ctx, TbResource resource, EntityExportData<TbResource> exportData) throws ThingsboardException {
super.setAdditionalExportData(ctx, resource, exportData);
resource.setPreview(null); // will be generated on import
}
@Override
public Set<EntityType> getSupportedEntityTypes() {
return Set.of(EntityType.TB_RESOURCE);

7
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/NotificationTargetImportService.java

@ -41,7 +41,7 @@ import org.thingsboard.server.dao.service.ConstraintValidator;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import java.util.stream.Collectors;
import java.util.List;
@Service
@TbCoreComponent
@ -66,10 +66,7 @@ public class NotificationTargetImportService extends BaseEntityImportService<Not
break;
case USER_LIST:
UserListFilter userListFilter = (UserListFilter) usersFilter;
userListFilter.setUsersIds(userListFilter.getUsersIds().stream()
.map(UserId::new).map(idProvider::getInternalId)
.map(UUIDBased::getId).collect(Collectors.toList())
);
userListFilter.setUsersIds(List.of(ctx.getUser().getUuidId())); // user entities are not supported by VC; replacing with current user id
break;
case TENANT_ADMINISTRATORS:
if (CollectionUtils.isNotEmpty(((TenantAdministratorsFilter) usersFilter).getTenantsIds()) ||

12
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/ResourceImportService.java

@ -18,12 +18,14 @@ package org.thingsboard.server.service.sync.ie.importing.impl;
import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.TbResource;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.TbResourceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.resource.ImageService;
import org.thingsboard.server.dao.resource.ResourceService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@ -34,6 +36,7 @@ import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
public class ResourceImportService extends BaseEntityImportService<TbResourceId, TbResource, EntityExportData<TbResource>> {
private final ResourceService resourceService;
private final ImageService imageService;
@Override
protected void setOwner(TenantId tenantId, TbResource resource, IdProvider idProvider) {
@ -66,7 +69,14 @@ public class ResourceImportService extends BaseEntityImportService<TbResourceId,
@Override
protected TbResource saveOrUpdate(EntitiesImportCtx ctx, TbResource resource, EntityExportData<TbResource> exportData, IdProvider idProvider) {
return resourceService.saveResource(resource);
if (resource.getResourceType() == ResourceType.IMAGE) {
return new TbResource(imageService.saveImage(resource));
} else {
resource = resourceService.saveResource(resource);
resource.setData(null);
resource.setPreview(null);
return resource;
}
}
@Override

46
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -17,19 +17,16 @@ package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.TbStopWatch;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
@ -72,6 +69,7 @@ import org.thingsboard.server.dao.DaoUtil;
import org.thingsboard.server.dao.exception.DeviceCredentialsValidationException;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.executors.VersionControlExecutor;
import org.thingsboard.server.service.sync.ie.EntitiesExportImportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException;
@ -85,8 +83,6 @@ import org.thingsboard.server.service.sync.vc.data.ReimportTask;
import org.thingsboard.server.service.sync.vc.data.SimpleEntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.repository.TbRepositorySettingsService;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collections;
@ -115,23 +111,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private final TbNotificationEntityService entityNotificationService;
private final TransactionTemplate transactionTemplate;
private final TbTransactionalCache<UUID, VersionControlTaskCacheEntry> taskCache;
private ListeningExecutorService executor;
@Value("${vc.thread_pool_size:4}")
private int threadPoolSize;
@PostConstruct
public void init() {
executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, DefaultEntitiesVersionControlService.class));
}
@PreDestroy
public void shutdown() {
if (executor != null) {
executor.shutdownNow();
}
}
private final VersionControlExecutor executor;
@SuppressWarnings("UnstableApiUsage")
@Override
@ -185,7 +165,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
throw new ThingsboardException(ThingsboardErrorCode.ITEM_NOT_FOUND);
} else {
var entry = cacheEntry.get();
log.debug("[{}] Cache get: {}", requestId, entry);
log.trace("[{}] Cache get: {}", requestId, entry);
var result = getter.apply(entry);
if (result == null) {
throw new ThingsboardException(ThingsboardErrorCode.BAD_REQUEST_PARAMS);
@ -287,7 +267,15 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private <R> VersionLoadResult doInTemplate(EntitiesImportCtx ctx, VersionLoadRequest request, Function<EntitiesImportCtx, VersionLoadResult> function) {
try {
VersionLoadResult result = transactionTemplate.execute(status -> function.apply(ctx));
VersionLoadResult result = transactionTemplate.execute(status -> {
try {
return function.apply(ctx);
} catch (RuntimeException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e); // to prevent UndeclaredThrowableException
}
});
for (ThrowingRunnable throwingRunnable : ctx.getEventCallbacks()) {
throwingRunnable.run();
}
@ -355,9 +343,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
sw.stop();
for (var task : sw.getTaskInfo()) {
log.info("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
log.debug("[{}] Executed: {} in {}ms", ctx.getTenantId(), task.getTaskName(), task.getTimeMillis());
}
log.info("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
log.debug("[{}] Total time: {}ms", ctx.getTenantId(), sw.getTotalTimeMillis());
return VersionLoadResult.success(new ArrayList<>(ctx.getResults().values()));
}
@ -380,8 +368,8 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
do {
try {
entityDataList = gitServiceQueue.getEntities(ctx.getTenantId(), ctx.getVersionId(), entityType, offset, limit).get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
} catch (ExecutionException e) {
throw e.getCause();
}
log.debug("[{}] Loading {} entities pack ({})", ctx.getTenantId(), entityType, entityDataList.size());
for (EntityExportData entityData : entityDataList) {
@ -623,7 +611,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
}
private VersionLoadResult cachePut(UUID requestId, VersionLoadResult result) {
log.debug("[{}] Cache put: {}", requestId, result);
log.trace("[{}] Cache put: {}", requestId, result);
taskCache.put(requestId, VersionControlTaskCacheEntry.newForImport(result));
return result;
}

172
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java

@ -18,7 +18,6 @@ package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import com.google.protobuf.ByteString;
import lombok.SneakyThrows;
@ -62,6 +61,7 @@ import org.thingsboard.server.queue.discovery.TbServiceInfoProvider;
import org.thingsboard.server.queue.scheduler.SchedulerComponent;
import org.thingsboard.server.queue.util.DataDecodingEncodingService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.executors.VersionControlExecutor;
import org.thingsboard.server.service.sync.vc.data.ClearRepositoryGitRequest;
import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import org.thingsboard.server.service.sync.vc.data.EntitiesContentGitRequest;
@ -92,6 +92,7 @@ import java.util.stream.Collectors;
@TbCoreComponent
@Service
@Slf4j
@SuppressWarnings("UnstableApiUsage")
public class DefaultGitVersionControlQueueService implements GitVersionControlQueueService {
private final TbServiceInfoProvider serviceInfoProvider;
@ -99,40 +100,42 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
private final DataDecodingEncodingService encodingService;
private final DefaultEntitiesVersionControlService entitiesVersionControlService;
private final SchedulerComponent scheduler;
private final VersionControlExecutor executor;
private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>();
private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new ConcurrentHashMap<>();
private final Map<UUID, HashMap<Integer, String[]>> chunkedMsgs = new ConcurrentHashMap<>();
@Value("${queue.vc.request-timeout:180000}")
private int requestTimeout;
@Value("${queue.vc.msg-chunk-size:500000}")
@Value("${queue.vc.msg-chunk-size:250000}")
private int msgChunkSize;
public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService,
DataDecodingEncodingService encodingService,
@Lazy DefaultEntitiesVersionControlService entitiesVersionControlService,
SchedulerComponent scheduler) {
SchedulerComponent scheduler, VersionControlExecutor executor) {
this.serviceInfoProvider = serviceInfoProvider;
this.clusterService = clusterService;
this.encodingService = encodingService;
this.entitiesVersionControlService = entitiesVersionControlService;
this.scheduler = scheduler;
this.executor = executor;
}
@Override
public ListenableFuture<CommitGitRequest> prepareCommit(User user, VersionCreateRequest request) {
SettableFuture<CommitGitRequest> future = SettableFuture.create();
log.debug("Executing prepareCommit [{}][{}]", request.getBranch(), request.getVersionName());
CommitGitRequest commit = new CommitGitRequest(user.getTenantId(), request);
registerAndSend(commit, builder -> builder.setCommitRequest(
ListenableFuture<Void> future = registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setPrepareMsg(getCommitPrepareMsg(user, request)).build()
).build(), wrap(future, commit));
return future;
).build());
return Futures.transform(future, f -> commit, executor);
}
@SuppressWarnings("UnstableApiUsage")
@SneakyThrows
@Override
public ListenableFuture<Void> addToCommit(CommitGitRequest commit, EntityExportData<ExportableEntity<EntityId>> entityData) {
log.debug("Executing addToCommit [{}][{}][{}]", entityData.getEntityType(), entityData.getEntity().getId(), commit.getRequestId());
String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId());
String entityDataJson = JacksonUtil.toPrettyString(entityData.sort());
@ -143,53 +146,42 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
AtomicInteger chunkIndex = new AtomicInteger();
List<ListenableFuture<Void>> futures = new ArrayList<>();
entityDataChunks.forEach(chunk -> {
SettableFuture<Void> chunkFuture = SettableFuture.create();
log.trace("[{}] sending chunk {} for 'addToCommit'", chunkedMsgId, chunkIndex.get());
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setAddMsg(
TransportProtos.AddMsg.newBuilder()
.setRelativePath(path).setEntityDataJsonChunk(chunk)
.setChunkedMsgId(chunkedMsgId).setChunkIndex(chunkIndex.getAndIncrement())
.setChunksCount(chunksCount).build()
ListenableFuture<Void> chunkFuture = registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setAddMsg(TransportProtos.AddMsg.newBuilder()
.setRelativePath(path).setEntityDataJsonChunk(chunk)
.setChunkedMsgId(chunkedMsgId).setChunkIndex(chunkIndex.getAndIncrement())
.setChunksCount(chunksCount)
).build()
).build(), wrap(chunkFuture, null));
).build());
futures.add(chunkFuture);
});
return Futures.transform(Futures.allAsList(futures), r -> {
log.trace("[{}] sent all chunks for 'addToCommit'", chunkedMsgId);
return null;
}, MoreExecutors.directExecutor());
}, executor);
}
@Override
public ListenableFuture<Void> deleteAll(CommitGitRequest commit, EntityType entityType) {
SettableFuture<Void> future = SettableFuture.create();
log.debug("Executing deleteAll [{}][{}][{}]", commit.getTenantId(), entityType, commit.getRequestId());
String path = getRelativePath(entityType, null);
registerAndSend(commit, builder -> builder.setCommitRequest(
return registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setDeleteMsg(
TransportProtos.DeleteMsg.newBuilder().setRelativePath(path).build()
).build()
).build(), wrap(future, null));
return future;
TransportProtos.DeleteMsg.newBuilder().setRelativePath(path)
)).build());
}
@Override
public ListenableFuture<VersionCreationResult> push(CommitGitRequest commit) {
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setPushMsg(
TransportProtos.PushMsg.newBuilder().build()
).build()
).build(), wrap(commit.getFuture()));
return commit.getFuture();
log.debug("Executing push [{}][{}]", commit.getTenantId(), commit.getRequestId());
return sendRequest(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setPushMsg(TransportProtos.PushMsg.getDefaultInstance())
));
}
@Override
public ListenableFuture<PageData<EntityVersion>> listVersions(TenantId tenantId, String branch, PageLink pageLink) {
return listVersions(tenantId,
applyPageLinkParameters(
ListVersionsRequestMsg.newBuilder()
@ -284,90 +276,95 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
@Override
@SuppressWarnings("rawtypes")
public ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId) {
log.debug("Executing getEntity [{}][{}][{}]", tenantId, versionId, entityId);
EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId);
chunkedMsgs.put(request.getRequestId(), new HashMap<>());
registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder()
.setVersionId(versionId)
.setEntityType(entityId.getEntityType().name())
.setEntityIdMSB(entityId.getId().getMostSignificantBits())
.setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build()
, wrap(request.getFuture()));
return request.getFuture();
return sendRequest(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder()
.setVersionId(versionId)
.setEntityType(entityId.getEntityType().name())
.setEntityIdMSB(entityId.getId().getMostSignificantBits())
.setEntityIdLSB(entityId.getId().getLeastSignificantBits())).build());
}
private <T> void registerAndSend(PendingGitRequest<T> request,
Function<ToVersionControlServiceMsg.Builder, ToVersionControlServiceMsg> enrichFunction, TbQueueCallback callback) {
registerAndSend(request, enrichFunction, null, callback);
private <T> ListenableFuture<Void> registerAndSend(PendingGitRequest<T> request,
Function<ToVersionControlServiceMsg.Builder, ToVersionControlServiceMsg> enrichFunction) {
return registerAndSend(request, enrichFunction, null);
}
private <T> void registerAndSend(PendingGitRequest<T> request,
Function<ToVersionControlServiceMsg.Builder, ToVersionControlServiceMsg> enrichFunction, RepositorySettings settings, TbQueueCallback callback) {
private <T> ListenableFuture<Void> registerAndSend(PendingGitRequest<T> request,
Function<ToVersionControlServiceMsg.Builder, ToVersionControlServiceMsg> enrichFunction,
RepositorySettings settings) {
if (!request.getFuture().isDone()) {
pendingRequestMap.putIfAbsent(request.getRequestId(), request);
var requestBody = enrichFunction.apply(newRequestProto(request, settings));
log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody);
clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback);
SettableFuture<Void> submitFuture = SettableFuture.create();
clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
submitFuture.set(null);
}
@Override
public void onFailure(Throwable t) {
submitFuture.setException(t);
}
});
if (request.getTimeoutTask() == null) {
request.setTimeoutTask(scheduler.schedule(() -> processTimeout(request.getRequestId()), requestTimeout, TimeUnit.MILLISECONDS));
}
return submitFuture;
} else {
throw new RuntimeException("Future is already done!");
}
}
private <T> ListenableFuture<T> sendRequest(PendingGitRequest<T> request, Consumer<ToVersionControlServiceMsg.Builder> enrichFunction) {
registerAndSend(request, builder -> {
return sendRequest(request, enrichFunction, null);
}
private <T> ListenableFuture<T> sendRequest(PendingGitRequest<T> request, Consumer<ToVersionControlServiceMsg.Builder> enrichFunction, RepositorySettings settings) {
ListenableFuture<Void> submitFuture = registerAndSend(request, builder -> {
enrichFunction.accept(builder);
return builder.build();
}, wrap(request.getFuture()));
return request.getFuture();
}, settings);
return Futures.transformAsync(submitFuture, input -> request.getFuture(), executor);
}
@Override
@SuppressWarnings("rawtypes")
public ListenableFuture<List<EntityExportData>> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) {
log.debug("Executing getEntities [{}][{}][{}]", tenantId, versionId, entityType);
EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType);
chunkedMsgs.put(request.getRequestId(), new HashMap<>());
registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder()
return sendRequest(request, builder -> builder.setEntitiesContentRequest(
EntitiesContentRequestMsg.newBuilder()
.setVersionId(versionId)
.setEntityType(entityType.name())
.setOffset(offset)
.setLimit(limit)
).build()
, wrap(request.getFuture()));
return request.getFuture();
).build());
}
@Override
public ListenableFuture<Void> initRepository(TenantId tenantId, RepositorySettings settings) {
log.debug("Executing initRepository [{}]", tenantId);
VoidGitRequest request = new VoidGitRequest(tenantId);
registerAndSend(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build()
, settings, wrap(request.getFuture()));
return request.getFuture();
return sendRequest(request, builder -> builder.setInitRepositoryRequest(GenericRepositoryRequestMsg.getDefaultInstance()), settings);
}
@Override
public ListenableFuture<Void> testRepository(TenantId tenantId, RepositorySettings settings) {
log.debug("Executing testRepository [{}]", tenantId);
VoidGitRequest request = new VoidGitRequest(tenantId);
registerAndSend(request, builder -> builder
.setTestRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build()
, settings, wrap(request.getFuture()));
return request.getFuture();
return sendRequest(request, builder -> builder.setTestRepositoryRequest(GenericRepositoryRequestMsg.getDefaultInstance()), settings);
}
@Override
public ListenableFuture<Void> clearRepository(TenantId tenantId) {
log.debug("Executing clearRepository [{}]", tenantId);
ClearRepositoryGitRequest request = new ClearRepositoryGitRequest(tenantId);
registerAndSend(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.newBuilder().build()).build()
, wrap(request.getFuture()));
return request.getFuture();
return sendRequest(request, builder -> builder.setClearRepositoryRequest(GenericRepositoryRequestMsg.getDefaultInstance()));
}
@Override
@ -518,35 +515,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
return JacksonUtil.fromString(data, EntityExportData.class);
}
//The future will be completed when the corresponding result arrives from kafka
private static <T> TbQueueCallback wrap(SettableFuture<T> future) {
return new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
}
@Override
public void onFailure(Throwable t) {
future.setException(t);
}
};
}
//The future will be completed when the request is successfully sent to kafka
private <T> TbQueueCallback wrap(SettableFuture<T> future, T value) {
return new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {
future.set(value);
}
@Override
public void onFailure(Throwable t) {
future.setException(t);
}
};
}
private static String getRelativePath(EntityType entityType, EntityId entityId) {
String path = entityType.name().toLowerCase();
if (entityId != null) {

5
application/src/main/java/org/thingsboard/server/service/update/DefaultUpdateService.java

@ -41,7 +41,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@ -81,8 +80,6 @@ public class DefaultUpdateService implements UpdateService {
private final RestTemplate restClient = new RestTemplate();
private UpdateMessage updateMessage;
private EdgeUpgradeMessage edgeUpgradeMessage;
private String edgeInstallVersion;
private String platform;
private String version;
@ -94,7 +91,6 @@ public class DefaultUpdateService implements UpdateService {
updateMessage = new UpdateMessage(false, version, "", "",
"https://thingsboard.io/docs/reference/releases",
"https://thingsboard.io/docs/reference/releases");
edgeUpgradeMessage = new EdgeUpgradeMessage(new HashMap<>());
if (updatesEnabled) {
try {
platform = System.getProperty("platform", "unknown");
@ -173,5 +169,4 @@ public class DefaultUpdateService implements UpdateService {
public UpdateMessage checkUpdates() {
return updateMessage;
}
}

10
application/src/main/resources/thingsboard.yml

@ -699,8 +699,8 @@ spring.resources.chain:
content:
enabled: "true" # This property enables or disables the content Version Strategy. This strategy allows Spring to generate a unique version for static resources, which is based on the content of the resource
spring.servlet.multipart.max-file-size: "50MB" # Total file size cannot exceed 50MB when configuring file uploads
spring.servlet.multipart.max-request-size: "50MB" # Total request size for a multipart/form-data cannot exceed 50MB
spring.servlet.multipart.max-file-size: "${SPRING_SERVLET_MULTIPART_MAX_FILE_SIZE:50MB}" # Total file size cannot exceed 50MB when configuring file uploads
spring.servlet.multipart.max-request-size: "${SPRING_SERVLET_MULTIPART_MAX_REQUEST_SIZE:50MB}" # Total request size for a multipart/form-data cannot exceed 50MB
spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation: "true" #Fix Postgres JPA Error (Method org.postgresql.jdbc.PgConnection.createClob() is not yet implemented)
spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PROPERTIES_HIBERNATE_ORDER_BY_DEFAULT_NULL_ORDERING:last}" # Note: as for current Spring JPA version, custom NullHandling for the Sort.Order is ignored and this parameter is used
@ -1379,7 +1379,7 @@ queue:
# Kafka properties for OTA updates topic
ota-updates: "${TB_QUEUE_KAFKA_OTA_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}"
# Kafka properties for Version Control topic
version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:10;min.insync.replicas:1}"
version-control: "${TB_QUEUE_KAFKA_VC_TOPIC_PROPERTIES:retention.ms:604800000;segment.bytes:26214400;retention.bytes:1048576000;partitions:1;min.insync.replicas:1}"
consumer-stats:
# Prints lag between consumer group offset and last messages offset in Kafka topics
enabled: "${TB_QUEUE_KAFKA_CONSUMER_STATS_ENABLED:true}"
@ -1398,6 +1398,8 @@ queue:
region: "${TB_QUEUE_AWS_SQS_REGION:YOUR_REGION}"
# Number of threads per each AWS SQS queue in consumer
threads_per_topic: "${TB_QUEUE_AWS_SQS_THREADS_PER_TOPIC:1}"
# Thread pool size for aws_sqs queue producer executor provider. Default value equals to AmazonSQSAsyncClient.DEFAULT_THREAD_POOL_SIZE
producer_thread_pool_size: "${TB_QUEUE_AWS_SQS_EXECUTOR_THREAD_POOL_SIZE:50}"
queue-properties:
# AWS SQS queue properties. VisibilityTimeout in seconds;MaximumMessageSize in bytes;MessageRetentionPeriod in seconds
rule-engine: "${TB_QUEUE_AWS_SQS_RE_QUEUE_PROPERTIES:VisibilityTimeout:30;MaximumMessageSize:262144;MessageRetentionPeriod:604800}"
@ -1672,7 +1674,7 @@ metrics:
# Version control parameters
vc:
# Pool size for handling export tasks
thread_pool_size: "${TB_VC_POOL_SIZE:2}"
thread_pool_size: "${TB_VC_POOL_SIZE:6}"
git:
# Pool size for handling the git IO operations
io_pool_size: "${TB_VC_GIT_POOL_SIZE:3}"

178
application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java

@ -17,6 +17,7 @@ package org.thingsboard.server.controller;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.NullNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.After;
import org.junit.Assert;
@ -663,6 +664,101 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest {
credentials.getCredentialsId()));
}
@Test
public void testFetchPublishTelemetryCommandsForDefaultDeviceIfPortsAndHostsNull() throws Exception {
loginSysAdmin();
ObjectNode config = JacksonUtil.newObjectNode();
ObjectNode http = JacksonUtil.newObjectNode();
http.put("enabled", true);
http.set("host", null);
http.set("port", null);
config.set("http", http);
ObjectNode https = JacksonUtil.newObjectNode();
https.put("enabled", true);
https.set("host", null);
https.set("port", null);
config.set("https", https);
ObjectNode mqtt = JacksonUtil.newObjectNode();
mqtt.put("enabled", true);
mqtt.set("host", null);
mqtt.set("port", null);
config.set("mqtt", mqtt);
ObjectNode mqtts = JacksonUtil.newObjectNode();
mqtts.put("enabled", true);
mqtts.set("host", null);
mqtts.set("port", null);
config.set("mqtts", mqtts);
ObjectNode coap = JacksonUtil.newObjectNode();
coap.put("enabled", true);
coap.set("host", null);
coap.set("port", null);
config.set("coap", coap);
ObjectNode coaps = JacksonUtil.newObjectNode();
coaps.put("enabled", true);
coaps.set("host", null);
coaps.set("port", null);
config.set("coaps", coaps);
AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class);
adminSettings.setJsonValue(config);
doPost("/api/admin/settings", adminSettings).andExpect(status().isOk());
login("tenant2@thingsboard.org", "testPassword1");
Device device = new Device();
device.setName("My device");
device.setType("default");
Device savedDevice = doPost("/api/device", device, Device.class);
JsonNode commands =
doGetTyped("/api/device-connectivity/" + savedDevice.getId().getId(), new TypeReference<>() {
});
DeviceCredentials credentials =
doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class);
assertThat(commands).hasSize(3);
JsonNode httpCommands = commands.get(HTTP);
assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost/api/v1/%s/telemetry " +
"--header Content-Type:application/json --data \"{temperature:25}\"",
credentials.getCredentialsId()));
assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost/api/v1/%s/telemetry " +
"--header Content-Type:application/json --data \"{temperature:25}\"",
credentials.getCredentialsId()));
JsonNode mqttCommands = commands.get(MQTT);
assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h localhost -t v1/devices/me/telemetry " +
"-u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o " + CA_ROOT_CERT_PEM + " http://localhost:80/api/device-connectivity/mqtts/certificate/download");
assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile " + CA_ROOT_CERT_PEM + " -h localhost " +
"-t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId()));
JsonNode dockerMqttCommands = commands.get(MQTT).get(DOCKER);
assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h host.docker.internal" +
" -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway thingsboard/mosquitto-clients " +
"/bin/sh -c \"curl -f -S -o " + CA_ROOT_CERT_PEM + " http://localhost:80/api/device-connectivity/mqtts/certificate/download && " +
"mosquitto_pub -d -q 1 --cafile " + CA_ROOT_CERT_PEM + " -h host.docker.internal -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", credentials.getCredentialsId()));
JsonNode linuxCoapCommands = commands.get(COAP);
assertThat(linuxCoapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://localhost/api/v1/%s/telemetry " +
"-t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(linuxCoapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://localhost/api/v1/%s/telemetry" +
" -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER);
assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway" +
" thingsboard/coap-clients coap-client -v 6 -m POST coap://host.docker.internal/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway" +
" thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://host.docker.internal/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
}
@Test
public void testFetchPublishTelemetryCommandsForDefaultDeviceIfHostIsNotLocalhost() throws Exception {
loginSysAdmin();
@ -722,6 +818,88 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest {
"thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://test.domain:5684/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
}
@Test
public void testFetchPublishTelemetryCommandsForDeviceWhenHostSetToNullInSettings() throws Exception {
loginSysAdmin();
ObjectNode config = JacksonUtil.newObjectNode();
ObjectNode http = JacksonUtil.newObjectNode();
http.put("enabled", true);
http.put("host", " ");
http.put("port", 8080);
config.set("http", http);
ObjectNode https = JacksonUtil.newObjectNode();
https.put("enabled", true);
https.put("host", "");
https.put("port", 443);
config.set("https", https);
ObjectNode mqtt = JacksonUtil.newObjectNode();
mqtt.put("enabled", true);
mqtt.set("host", NullNode.getInstance());
mqtt.set("port", NullNode.getInstance());
config.set("mqtt", mqtt);
ObjectNode mqtts = JacksonUtil.newObjectNode();
mqtts.put("enabled", true);
mqtts.put("host", "");
mqtts.set("port", NullNode.getInstance());
config.set("mqtts", mqtts);
ObjectNode coap = JacksonUtil.newObjectNode();
coap.put("enabled", true);
coap.set("host", NullNode.getInstance());
coap.put("port", "");
config.set("coap", coap);
ObjectNode coaps = JacksonUtil.newObjectNode();
coaps.put("enabled", true);
coaps.set("host", NullNode.getInstance());
coaps.set("port", NullNode.getInstance());
config.set("coaps", coaps);
AdminSettings adminSettings = doGet("/api/admin/settings/connectivity", AdminSettings.class);
adminSettings.setJsonValue(config);
doPost("/api/admin/settings", adminSettings).andExpect(status().isOk());
login("tenant2@thingsboard.org", "testPassword1");
Device device = new Device();
device.setName("My device");
device.setType("default");
Device savedDevice = doPost("/api/device", device, Device.class);
JsonNode commands =
doGetTyped("/api/device-connectivity/" + savedDevice.getId().getId(), new TypeReference<>() {
});
DeviceCredentials credentials =
doGet("/api/device/" + savedDevice.getId().getId() + "/credentials", DeviceCredentials.class);
assertThat(commands).hasSize(3);
JsonNode httpCommands = commands.get(HTTP);
assertThat(httpCommands.get(HTTP).asText()).isEqualTo(String.format("curl -v -X POST http://localhost:8080/api/v1/%s/telemetry --header Content-Type:application/json --data \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(httpCommands.get(HTTPS).asText()).isEqualTo(String.format("curl -v -X POST https://localhost/api/v1/%s/telemetry --header Content-Type:application/json --data \"{temperature:25}\"", credentials.getCredentialsId()));
JsonNode mqttCommands = commands.get(MQTT);
assertThat(mqttCommands.get(MQTT).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 -h localhost -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(mqttCommands.get(MQTTS).get(0).asText()).isEqualTo("curl -f -S -o " + CA_ROOT_CERT_PEM + " http://localhost:80/api/device-connectivity/mqtts/certificate/download");
assertThat(mqttCommands.get(MQTTS).get(1).asText()).isEqualTo(String.format("mosquitto_pub -d -q 1 --cafile " + CA_ROOT_CERT_PEM + " -h localhost -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId()));
JsonNode dockerMqttCommands = mqttCommands.get(DOCKER);
assertThat(dockerMqttCommands.get(MQTT).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway thingsboard/mosquitto-clients mosquitto_pub -d -q 1 -h host.docker.internal -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(dockerMqttCommands.get(MQTTS).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway thingsboard/mosquitto-clients /bin/sh -c \"curl -f -S -o " + CA_ROOT_CERT_PEM + " http://localhost:80/api/device-connectivity/mqtts/certificate/download && mosquitto_pub -d -q 1 --cafile " + CA_ROOT_CERT_PEM + " -h host.docker.internal -t v1/devices/me/telemetry -u \"%s\" -m \"{temperature:25}\"\"", credentials.getCredentialsId()));
JsonNode coapCommands = commands.get(COAP);
assertThat(coapCommands.get(COAP).asText()).isEqualTo(String.format("coap-client -v 6 -m POST coap://localhost/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(coapCommands.get(COAPS).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST coaps://localhost/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
JsonNode dockerCoapCommands = coapCommands.get(DOCKER);
assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway thingsboard/coap-clients coap-client -v 6 -m POST coap://host.docker.internal/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
assertThat(dockerCoapCommands.get(COAPS).asText()).isEqualTo(String.format("docker run --rm -it --add-host=host.docker.internal:host-gateway thingsboard/coap-clients coap-client-openssl -v 6 -m POST coaps://host.docker.internal/api/v1/%s/telemetry -t json -e \"{temperature:25}\"", credentials.getCredentialsId()));
}
private void setConnectivityHost(String host) throws Exception {
ObjectNode config = JacksonUtil.newObjectNode();

42
application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java

@ -44,11 +44,11 @@ import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EdgeUpgradeInfo;
import org.thingsboard.server.common.data.EntitySubtype;
import org.thingsboard.server.common.data.StringUtils;
import org.thingsboard.server.common.data.Tenant;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.EdgeUpgradeInfo;
import org.thingsboard.server.common.data.User;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.asset.AssetProfile;
@ -864,6 +864,8 @@ public class EdgeControllerTest extends AbstractControllerTest {
doPost("/api/admin/jwtSettings", settings).andExpect(status().isOk());
loginTenantAdmin();
Edge edge = doPost("/api/edge", constructEdge("Test Sync Edge", "test"), Edge.class);
Asset asset = new Asset();
asset.setName("Test Sync Edge Asset 1");
asset.setType("test");
@ -874,8 +876,6 @@ public class EdgeControllerTest extends AbstractControllerTest {
device.setType("default");
Device savedDevice = doPost("/api/device", device, Device.class);
Edge edge = doPost("/api/edge", constructEdge("Test Sync Edge", "test"), Edge.class);
simulateEdgeActivation(edge);
doPost("/api/edge/" + edge.getId().getId().toString()
@ -1200,4 +1200,40 @@ public class EdgeControllerTest extends AbstractControllerTest {
Assert.assertTrue(upgradeInstructions.contains("Upgrading to 3.6.1EDGE"));
Assert.assertTrue(upgradeInstructions.contains("Upgrading to 3.6.2EDGE"));
}
@Test
public void testIsEdgeUpgradeAvailable() throws Exception {
Edge edge = constructEdge("Edge Upgrade Available", "default");
Edge savedEdge = doPost("/api/edge", edge, Edge.class);
// Test 3.5.0 Edge - upgrade not available
String body = "{\"edgeVersion\": \"V_3_5_0\"}";
doPostAsync("/api/plugins/telemetry/EDGE/" + savedEdge.getId().getId() + "/attributes/SERVER_SCOPE", body, String.class, status().isOk());
edgeUpgradeInstructionsService.setAppVersion("3.6.0");
Assert.assertFalse(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
edgeUpgradeInstructionsService.setAppVersion("3.6.2");
Assert.assertFalse(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
edgeUpgradeInstructionsService.setAppVersion("3.6.2.7");
Assert.assertFalse(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
// Test 3.6.0 Edge - upgrade available
body = "{\"edgeVersion\": \"V_3_6_0\"}";
doPostAsync("/api/plugins/telemetry/EDGE/" + savedEdge.getId().getId() + "/attributes/SERVER_SCOPE", body, String.class, status().isOk());
edgeUpgradeInstructionsService.setAppVersion("3.6.0");
Assert.assertFalse(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
edgeUpgradeInstructionsService.setAppVersion("3.6.1.5");
Assert.assertTrue(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
edgeUpgradeInstructionsService.setAppVersion("3.6.2");
Assert.assertTrue(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
// Test 3.6.1 Edge - upgrade available
body = "{\"edgeVersion\": \"V_3_6_1\"}";
doPostAsync("/api/plugins/telemetry/EDGE/" + savedEdge.getId().getId() + "/attributes/SERVER_SCOPE", body, String.class, status().isOk());
edgeUpgradeInstructionsService.setAppVersion("3.6.1");
Assert.assertFalse(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
edgeUpgradeInstructionsService.setAppVersion("3.6.2");
Assert.assertTrue(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
edgeUpgradeInstructionsService.setAppVersion("3.6.2.6");
Assert.assertTrue(edgeUpgradeInstructionsService.isUpgradeAvailable(savedEdge.getTenantId(), savedEdge.getId()));
}
}

80
application/src/test/java/org/thingsboard/server/controller/ImageControllerTest.java

@ -23,12 +23,12 @@ import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpMethod;
import org.springframework.mock.web.MockMultipartFile;
import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.ImageDescriptor;
import org.thingsboard.server.common.data.ImageExportData;
import org.thingsboard.server.common.data.ResourceType;
import org.thingsboard.server.common.data.SystemParams;
import org.thingsboard.server.common.data.TbResourceInfo;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
@ -173,6 +173,8 @@ public class ImageControllerTest extends AbstractControllerTest {
assertThat(exportData.getTitle()).isEqualTo(filename);
assertThat(exportData.getResourceKey()).isEqualTo(filename);
assertThat(exportData.getData()).isEqualTo(Base64.getEncoder().encodeToString(PNG_IMAGE));
assertThat(exportData.isPublic()).isTrue();
assertThat(exportData.getPublicResourceKey()).isNotEmpty();
doDelete("/api/images/tenant/" + filename).andExpect(status().isOk());
@ -180,6 +182,8 @@ public class ImageControllerTest extends AbstractControllerTest {
assertThat(importedImageInfo.getTitle()).isEqualTo(filename);
assertThat(importedImageInfo.getResourceKey()).isEqualTo(filename);
assertThat(importedImageInfo.getFileName()).isEqualTo(filename);
assertThat(importedImageInfo.isPublic()).isTrue();
assertThat(importedImageInfo.getPublicResourceKey()).isEqualTo(exportData.getPublicResourceKey());
checkPngImageDescriptor(importedImageInfo.getDescriptor(ImageDescriptor.class));
assertThat(downloadImage("tenant", filename)).containsExactly(PNG_IMAGE);
}
@ -206,6 +210,65 @@ public class ImageControllerTest extends AbstractControllerTest {
.containsOnly(tenantImage);
}
@Test
public void testUploadPublicImage() throws Exception {
String filename = "my_public_image.png";
TbResourceInfo imageInfo = uploadImage(HttpMethod.POST, "/api/image", filename, "image/png", PNG_IMAGE);
assertThat(imageInfo.isPublic()).isTrue();
assertThat(imageInfo.getPublicResourceKey()).hasSize(32);
assertThat(imageInfo.getPublicLink()).isEqualTo("/api/images/public/" + imageInfo.getPublicResourceKey());
assertThat(downloadImage("tenant", filename)).containsExactly(PNG_IMAGE);
resetTokens();
assertThat(downloadPublicImage(imageInfo.getPublicResourceKey())).containsExactly(PNG_IMAGE);
}
@Test
public void testMakeImagePublic() throws Exception {
String filename = "my_public_image.png";
TbResourceInfo imageInfo = uploadImage(HttpMethod.POST, "/api/image", filename, "image/png", PNG_IMAGE);
String publicKey = imageInfo.getPublicResourceKey();
assertThat(publicKey).hasSize(32);
updateImagePublicStatus(filename, false);
doGet("/api/images/public/" + publicKey).andExpect(status().isNotFound());
updateImagePublicStatus(filename, true);
resetTokens();
assertThat(downloadPublicImage(publicKey)).containsExactly(PNG_IMAGE);
loginTenantAdmin();
updateImagePublicStatus(filename, false);
doGet("/api/images/public/" + publicKey).andExpect(status().isNotFound());
}
@Test
public void testGetImageUploadSpecs() throws Exception {
SystemParams systemParams = doGet("/api/system/params", SystemParams.class);
assertThat(systemParams.getMaxResourceSize()).isZero();
loginSysAdmin();
updateDefaultTenantProfileConfig(tenantProfileConfig -> {
tenantProfileConfig.setMaxResourceSize(100);
});
loginTenantAdmin();
systemParams = doGet("/api/system/params", SystemParams.class);
assertThat(systemParams.getMaxResourceSize()).isEqualTo(100);
loginSysAdmin();
updateDefaultTenantProfileConfig(tenantProfileConfig -> {
tenantProfileConfig.setMaxResourceSize(0);
});
loginTenantAdmin();
systemParams = doGet("/api/system/params", SystemParams.class);
assertThat(systemParams.getMaxResourceSize()).isEqualTo(0);
}
private TbResourceInfo updateImagePublicStatus(String filename, boolean isPublic) throws Exception {
return doPut("/api/images/tenant/" + filename + "/public/" + isPublic, "", TbResourceInfo.class);
}
private void checkPngImageDescriptor(ImageDescriptor imageDescriptor) {
assertThat(imageDescriptor.getMediaType()).isEqualTo("image/png");
assertThat(imageDescriptor.getWidth()).isEqualTo(200);
@ -242,13 +305,6 @@ public class ImageControllerTest extends AbstractControllerTest {
assertThat(imageDescriptor.getHeight()).isEqualTo(150);
assertThat(imageDescriptor.getSize()).isEqualTo(SVG_IMAGE.length);
assertThat(imageDescriptor.getEtag()).isEqualTo(Hashing.sha256().hashBytes(SVG_IMAGE).toString());
ImageDescriptor previewDescriptor = imageDescriptor.getPreviewDescriptor();
assertThat(previewDescriptor.getMediaType()).isEqualTo("image/png");
assertThat(previewDescriptor.getWidth()).isEqualTo(250);
assertThat(previewDescriptor.getHeight()).isEqualTo(250);
assertThat(previewDescriptor.getSize()).isEqualTo(13247);
assertThat(previewDescriptor.getEtag()).isEqualTo("f02db84f2c6ed7ea0606ae1145986a16a534cb0fd70436b3b5ca585569d48e46");
}
private List<TbResourceInfo> getImages(String searchText, boolean includeSystemImages, int limit) throws Exception {
@ -266,10 +322,14 @@ public class ImageControllerTest extends AbstractControllerTest {
.andReturn().getResponse().getContentAsByteArray();
}
private byte[] downloadPublicImage(String publicKey) throws Exception {
return doGet("/api/images/public/" + publicKey).andExpect(status().isOk())
.andReturn().getResponse().getContentAsByteArray();
}
private <R> TbResourceInfo uploadImage(HttpMethod httpMethod, String url, String filename, String mediaType, byte[] content) throws Exception {
MockMultipartFile file = new MockMultipartFile("file", filename, mediaType, content);
MockMultipartHttpServletRequestBuilder request = MockMvcRequestBuilders.multipart(httpMethod, url);
request.file(file);
var request = MockMvcRequestBuilders.multipart(httpMethod, url).file(file);
setJwtToken(request);
return readResponse(mockMvc.perform(request).andExpect(status().isOk()), TbResourceInfo.class);
}

2
common/actor/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/cache/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/cluster-api/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/coap-server/pom.xml

@ -22,7 +22,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/dao-api/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/dao-api/src/main/java/org/thingsboard/server/dao/resource/ImageService.java

@ -34,6 +34,8 @@ public interface ImageService {
TbResourceInfo getImageInfoByTenantIdAndKey(TenantId tenantId, String key);
TbResourceInfo getPublicImageInfoByKey(String publicResourceKey);
PageData<TbResourceInfo> getImagesByTenantId(TenantId tenantId, PageLink pageLink);
PageData<TbResourceInfo> getAllImagesByTenantId(TenantId tenantId, PageLink pageLink);

1
common/dao-api/src/main/java/org/thingsboard/server/dao/rule/RuleChainService.java

@ -100,6 +100,7 @@ public interface RuleChainService extends EntityDaoService {
PageData<RuleNode> findAllRuleNodesByType(String type, PageLink pageLink);
@Deprecated(forRemoval = true, since = "3.6.3")
PageData<RuleNode> findAllRuleNodesByTypeAndVersionLessThan(String type, int version, PageLink pageLink);
PageData<RuleNodeId> findAllRuleNodeIdsByTypeAndVersionLessThan(String type, int version, PageLink pageLink);

2
common/data/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

1
common/data/src/main/java/org/thingsboard/server/common/data/DataConstants.java

@ -124,6 +124,7 @@ public class DataConstants {
public static final String PASSWORD = "password";
public static final String EDGE_MSG_SOURCE = "edge";
public static final String MSG_SOURCE_KEY = "source";
public static final String EDGE_VERSION_ATTR_KEY = "edgeVersion";
public static final String LAST_CONNECTED_GATEWAY = "lastConnectedGateway";

18
common/data/src/main/java/org/thingsboard/server/common/data/ImageExportData.java

@ -16,18 +16,26 @@
package org.thingsboard.server.common.data;
import io.swagger.annotations.ApiModel;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@ApiModel
@Slf4j
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class ImageExportData {
private final String mediaType;
private final String fileName;
private final String title;
private final String resourceKey;
private final String data;
private String mediaType;
private String fileName;
private String title;
private String resourceKey;
private boolean isPublic;
private String publicResourceKey;
private String data;
}

1
common/data/src/main/java/org/thingsboard/server/common/data/SystemParams.java

@ -30,4 +30,5 @@ public class SystemParams {
boolean persistDeviceStateToTelemetry;
JsonNode userSettings;
long maxDatapointsLimit;
long maxResourceSize;
}

15
common/data/src/main/java/org/thingsboard/server/common/data/TbResourceInfo.java

@ -51,6 +51,8 @@ public class TbResourceInfo extends BaseData<TbResourceId> implements HasName, H
@Length(fieldName = "resourceKey")
@ApiModelProperty(position = 6, value = "Resource key.", example = "19_1.0", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private String resourceKey;
private boolean isPublic;
private String publicResourceKey;
@ApiModelProperty(position = 7, value = "Resource search text.", example = "19_1.0:binaryappdatacontainer", accessMode = ApiModelProperty.AccessMode.READ_ONLY)
private String searchText;
@ -79,6 +81,8 @@ public class TbResourceInfo extends BaseData<TbResourceId> implements HasName, H
this.resourceType = resourceInfo.resourceType;
this.resourceKey = resourceInfo.resourceKey;
this.searchText = resourceInfo.searchText;
this.isPublic = resourceInfo.isPublic;
this.publicResourceKey = resourceInfo.publicResourceKey;
this.etag = resourceInfo.etag;
this.fileName = resourceInfo.fileName;
this.descriptor = resourceInfo.descriptor != null ? resourceInfo.descriptor.deepCopy() : null;
@ -109,7 +113,16 @@ public class TbResourceInfo extends BaseData<TbResourceId> implements HasName, H
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
public String getLink() {
if (resourceType == ResourceType.IMAGE) {
return "/api/images/" + ((tenantId == null || !tenantId.isSysTenantId()) ? "tenant" : "system") + "/" + resourceKey; // tenantId is null in case of export to git
String type = (tenantId != null && tenantId.isSysTenantId()) ? "system" : "tenant"; // tenantId is null in case of export to git
return "/api/images/" + type + "/" + resourceKey;
}
return null;
}
@JsonProperty(access = JsonProperty.Access.READ_ONLY)
public String getPublicLink() {
if (resourceType == ResourceType.IMAGE && isPublic) {
return "/api/images/public/" + getPublicResourceKey();
}
return null;
}

23
common/data/src/main/java/org/thingsboard/server/common/data/util/ThrowingSupplier.java

@ -0,0 +1,23 @@
/**
* Copyright © 2016-2023 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.util;
@FunctionalInterface
public interface ThrowingSupplier<T> {
T get() throws Exception;
}

2
common/edge-api/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/message/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>thingsboard</artifactId>
</parent>
<artifactId>common</artifactId>

2
common/proto/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

7
common/proto/src/main/proto/queue.proto

@ -309,7 +309,12 @@ message CoreStartupMsg {
message ResourceCacheInvalidateMsg {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
string resourceKey = 3;
repeated ImageCacheKeyProto keys = 3;
}
message ImageCacheKeyProto {
optional string resourceKey = 1;
optional string publicResourceKey = 2;
}
message LwM2MRegistrationRequestMsg {

2
common/queue/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

12
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsAdmin.java

@ -24,11 +24,17 @@ import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.services.sqs.model.CreateQueueRequest;
import com.amazonaws.services.sqs.model.GetQueueUrlResult;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.common.util.ExecutorProvider;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.queue.TbQueueAdmin;
import org.thingsboard.server.queue.util.PropertyUtils;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -38,6 +44,8 @@ public class TbAwsSqsAdmin implements TbQueueAdmin {
private final Map<String, String> attributes;
private final AmazonSQS sqsClient;
private final Map<String, String> queues;
@Getter
private final ExecutorService producerExecutor;
public TbAwsSqsAdmin(TbAwsSqsSettings sqsSettings, Map<String, String> attributes) {
this.attributes = attributes;
@ -49,6 +57,7 @@ public class TbAwsSqsAdmin implements TbQueueAdmin {
AWSCredentials awsCredentials = new BasicAWSCredentials(sqsSettings.getAccessKeyId(), sqsSettings.getSecretAccessKey());
credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
}
producerExecutor = ThingsBoardExecutors.newWorkStealingPool(sqsSettings.getThreadPoolSize(), "aws-sqs-queue-executor");
sqsClient = AmazonSQSClientBuilder.standard()
.withCredentials(credentialsProvider)
@ -104,5 +113,8 @@ public class TbAwsSqsAdmin implements TbQueueAdmin {
if (sqsClient != null) {
sqsClient.shutdown();
}
if (producerExecutor != null) {
producerExecutor.shutdownNow();
}
}
}

42
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsProducerTemplate.java

@ -20,15 +20,11 @@ import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
import com.amazonaws.services.sqs.AmazonSQS;
import com.amazonaws.services.sqs.AmazonSQSClientBuilder;
import com.amazonaws.handlers.AsyncHandler;
import com.amazonaws.services.sqs.AmazonSQSAsync;
import com.amazonaws.services.sqs.AmazonSQSAsyncClientBuilder;
import com.amazonaws.services.sqs.model.SendMessageRequest;
import com.amazonaws.services.sqs.model.SendMessageResult;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.gson.Gson;
import lombok.extern.slf4j.Slf4j;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
@ -41,19 +37,17 @@ import org.thingsboard.server.queue.common.DefaultTbQueueMsg;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
@Slf4j
public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueueProducer<T> {
private final String defaultTopic;
private final AmazonSQS sqsClient;
private final AmazonSQSAsync sqsClient;
private final Gson gson = new Gson();
private final Map<String, String> queueUrlMap = new ConcurrentHashMap<>();
private final TbQueueAdmin admin;
private ListeningExecutorService producerExecutor;
private final TbAwsSqsAdmin admin;
public TbAwsSqsProducerTemplate(TbQueueAdmin admin, TbAwsSqsSettings sqsSettings, String defaultTopic) {
this.admin = admin;
this.admin = (TbAwsSqsAdmin) admin;
this.defaultTopic = defaultTopic;
AWSCredentialsProvider credentialsProvider;
@ -64,11 +58,11 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
credentialsProvider = new AWSStaticCredentialsProvider(awsCredentials);
}
sqsClient = AmazonSQSClientBuilder.standard()
sqsClient = AmazonSQSAsyncClientBuilder.standard()
.withCredentials(credentialsProvider)
.withRegion(sqsSettings.getRegion())
.withExecutorFactory(this.admin::getProducerExecutor)
.build();
producerExecutor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());
}
@Override
@ -91,30 +85,24 @@ public class TbAwsSqsProducerTemplate<T extends TbQueueMsg> implements TbQueuePr
sendMsgRequest.withMessageGroupId(sqsMsgId);
sendMsgRequest.withMessageDeduplicationId(sqsMsgId);
ListenableFuture<SendMessageResult> future = producerExecutor.submit(() -> sqsClient.sendMessage(sendMsgRequest));
Futures.addCallback(future, new FutureCallback<SendMessageResult>() {
@Override
public void onSuccess(SendMessageResult result) {
sqsClient.sendMessageAsync(sendMsgRequest, new AsyncHandler<SendMessageRequest, SendMessageResult>() {
@Override public void onError(Exception e) {
if (callback != null) {
callback.onSuccess(new AwsSqsTbQueueMsgMetadata(result.getSdkHttpMetadata()));
callback.onFailure(e);
}
}
@Override
public void onFailure(Throwable t) {
@Override public void onSuccess(SendMessageRequest request,
SendMessageResult sendMessageResult) {
if (callback != null) {
callback.onFailure(t);
callback.onSuccess(new AwsSqsTbQueueMsgMetadata(sendMessageResult.getSdkHttpMetadata()));
}
}
}, producerExecutor);
});
}
@Override
public void stop() {
if (producerExecutor != null) {
producerExecutor.shutdownNow();
}
if (sqsClient != null) {
sqsClient.shutdown();
}

3
common/queue/src/main/java/org/thingsboard/server/queue/sqs/TbAwsSqsSettings.java

@ -42,4 +42,7 @@ public class TbAwsSqsSettings {
@Value("${queue.aws_sqs.threads_per_topic}")
private int threadsPerTopic;
@Value("${queue.aws_sqs.producer_thread_pool_size:50}")
private int threadPoolSize;
}

2
common/script/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/script/remote-js-client/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>script</artifactId>
</parent>
<groupId>org.thingsboard.common.script</groupId>

2
common/script/script-api/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>script</artifactId>
</parent>
<groupId>org.thingsboard.common.script</groupId>

21
common/script/script-api/src/test/java/org/thingsboard/script/api/tbel/TbDateTest.java

@ -177,8 +177,9 @@ class TbDateTest {
* For Java 17:
* `{ "AM", "PM" }`
*/
s = "09:15:30 nachm., So. 10/09/2022";
d = new TbDate(s, pattern, "de","Europe/Berlin");
String s_ver = Runtime.version().feature() == 11 ? "09:15:30 nachm., So. 10/09/2022" :
"09:15:30 PM, So. 10/09/2022";
d = new TbDate(s_ver, pattern, "de","Europe/Berlin");
Assert.assertEquals("2022-10-09T19:15:30Z", d.toISOString());
s = "02:15:30 пп, середа, 4 жовтня 2023 р.";
@ -295,7 +296,9 @@ class TbDateTest {
Assert.assertEquals("9/5/23, 9:04:05 PM", d.toLocaleString("en-US", "America/New_York"));
Assert.assertEquals("23. 9. 5. 오후 9:04:05", d.toLocaleString("ko-KR", "America/New_York"));
Assert.assertEquals("06.09.23, 04:04:05", d.toLocaleString( "uk-UA", "Europe/Kiev"));
Assert.assertEquals("5\u200F/9\u200F/2023 9:04:05 م", d.toLocaleString( "ar-EG", "America/New_York"));
String expected_ver = Runtime.version().feature() == 11 ? "5\u200F/9\u200F/2023 9:04:05 م" :
"5\u200F/9\u200F/2023, 9:04:05 م";
Assert.assertEquals(expected_ver, d.toLocaleString( "ar-EG", "America/New_York"));
Assert.assertEquals("Tuesday, September 5, 2023 at 9:04:05 PM Eastern Daylight Time", d.toLocaleString("en-US", JacksonUtil.newObjectNode()
.put("timeZone", "America/New_York")
@ -311,8 +314,11 @@ class TbDateTest {
.put("timeZone", "Europe/Kiev")
.put("dateStyle", "full")
.put("timeStyle", "full")
.toString()));
Assert.assertEquals("الثلاثاء، 5 سبتمبر 2023 9:04:05 م التوقيت الصيفي الشرقي لأمريكا الشمالية", d.toLocaleString("ar-EG", JacksonUtil.newObjectNode()
.toString()));
expected_ver = Runtime.version().feature() == 11 ? "الثلاثاء، 5 سبتمبر 2023 9:04:05 م التوقيت الصيفي الشرقي لأمريكا الشمالية" :
"الثلاثاء، 5 سبتمبر 2023 في 9:04:05 م التوقيت الصيفي الشرقي لأمريكا الشمالية";
Assert.assertEquals(expected_ver, d.toLocaleString("ar-EG", JacksonUtil.newObjectNode()
.put("timeZone", "America/New_York")
.put("dateStyle", "full")
.put("timeStyle", "full")
@ -381,8 +387,9 @@ class TbDateTest {
// With pattern + locale - ok
String pattern = "hh:mm:ss a, EEE M/d/uuuu";
stringDateRFC_1123 = "09:15:30 nachm., So. 10/09/2022";
d = new TbDate(stringDateRFC_1123 , pattern, "de");
String stringDate_ver_RFC_1123 = Runtime.version().feature() == 11 ? "09:15:30 nachm., So. 10/09/2022" :
"09:15:30 PM, So. 10/09/2022";
d = new TbDate(stringDate_ver_RFC_1123 , pattern, "de");
Assert.assertEquals("2022-10-09 21:15:30", d.toLocaleString());
// failed TZ

2
common/stats/pom.xml

@ -22,7 +22,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/transport/coap/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/http/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/lwm2m/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/mqtt/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/transport/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/transport/snmp/pom.xml

@ -21,7 +21,7 @@
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>

2
common/transport/transport-api/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard.common</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>transport</artifactId>
</parent>
<groupId>org.thingsboard.common.transport</groupId>

2
common/util/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

2
common/version-control/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>common</artifactId>
</parent>
<groupId>org.thingsboard.common</groupId>

15
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java

@ -24,6 +24,7 @@ import com.google.common.util.concurrent.MoreExecutors;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.eclipse.jgit.errors.LargeObjectException;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
@ -126,7 +127,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
private long packProcessingTimeout;
@Value("${vc.git.io_pool_size:3}")
private int ioPoolSize;
@Value("${queue.vc.msg-chunk-size:500000}")
@Value("${queue.vc.msg-chunk-size:250000}")
private int msgChunkSize;
//We need to manually manage the threads since tasks for particular tenant need to be processed sequentially.
@ -261,7 +262,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
}
} catch (Exception e) {
reply(ctx, Optional.of(e));
reply(ctx, Optional.of(handleError(e)));
} finally {
lock.unlock();
}
@ -303,6 +304,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException {
String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString());
log.debug("Executing handleEntityContentRequest [{}][{}]", ctx.getTenantId(), path);
String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId());
Iterable<String> dataChunks = StringUtils.split(data, msgChunkSize);
@ -393,6 +395,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception {
log.debug("Executing handleCommitRequest [{}][{}]", ctx.getTenantId(), ctx.getRequestId());
var tenantId = ctx.getTenantId();
UUID txId = UUID.fromString(request.getTxId());
if (request.hasPrepareMsg()) {
@ -443,6 +446,7 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
private void addToCommit(VersionControlRequestCtx ctx, PendingCommit commit, AddMsg addMsg) throws IOException {
log.debug("Executing addToCommit [{}][{}]", ctx.getTenantId(), ctx.getRequestId());
log.trace("[{}] received chunk {} for 'addToCommit'", addMsg.getChunkedMsgId(), addMsg.getChunkIndex());
Map<String, String[]> chunkedMsgs = commit.getChunkedMsgs();
String[] msgChunks = chunkedMsgs.computeIfAbsent(addMsg.getChunkedMsgId(), id -> new String[addMsg.getChunksCount()]);
@ -496,6 +500,13 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
}
private Exception handleError(Exception e) {
if (e instanceof LargeObjectException) {
return new RuntimeException("Version is too big");
}
return e;
}
private void reply(VersionControlRequestCtx ctx, VersionCreationResult result) {
var responseBuilder = CommitResponseMsg.newBuilder().setAdded(result.getAdded())
.setModified(result.getModified())

9
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@ -39,6 +39,7 @@ import org.eclipse.jgit.diff.EditList;
import org.eclipse.jgit.diff.HistogramDiff;
import org.eclipse.jgit.diff.RawText;
import org.eclipse.jgit.diff.RawTextComparator;
import org.eclipse.jgit.errors.LargeObjectException;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
@ -236,8 +237,12 @@ public class GitRepository {
ObjectId blobId = treeWalk.getObjectId(0);
try (ObjectReader objectReader = git.getRepository().newObjectReader()) {
ObjectLoader objectLoader = objectReader.open(blobId);
byte[] bytes = objectLoader.getBytes();
return new String(bytes, StandardCharsets.UTF_8);
try {
byte[] bytes = objectLoader.getBytes();
return new String(bytes, StandardCharsets.UTF_8);
} catch (LargeObjectException e) {
throw new RuntimeException("File " + file + " is too big to load");
}
}
}
}

2
dao/pom.xml

@ -20,7 +20,7 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.thingsboard</groupId>
<version>3.6.2-SNAPSHOT</version>
<version>3.6.3-SNAPSHOT</version>
<artifactId>thingsboard</artifactId>
</parent>
<artifactId>dao</artifactId>

13
dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java

@ -62,6 +62,7 @@ import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.HTTPS;
import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTT;
import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.MQTTS;
import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.getHost;
import static org.thingsboard.server.dao.util.DeviceConnectivityUtil.getPort;
@Service("DeviceConnectivityDaoService")
@Slf4j
@ -217,7 +218,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService
return null;
}
String hostName = getHost(baseUrl, properties, protocol);
String propertiesPort = properties.getPort();
String propertiesPort = getPort(properties);
String port = (propertiesPort.isEmpty() || HTTP_DEFAULT_PORT.equals(propertiesPort) || HTTPS_DEFAULT_PORT.equals(propertiesPort))
? "" : ":" + propertiesPort;
return DeviceConnectivityUtil.getHttpPublishCommand(protocol, hostName, port, deviceCredentials);
@ -265,14 +266,14 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService
private String getMqttPublishCommand(String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException {
DeviceConnectivityInfo properties = getConnectivity(MQTT);
String mqttHost = getHost(baseUrl, properties, MQTT);
String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort();
String mqttPort = getPort(properties);
return DeviceConnectivityUtil.getMqttPublishCommand(MQTT, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials);
}
private List<String> getMqttsPublishCommand(String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException {
DeviceConnectivityInfo properties = getConnectivity(MQTTS);
String mqttHost = getHost(baseUrl, properties, MQTTS);
String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort();
String mqttPort = getPort(properties);
String pubCommand = DeviceConnectivityUtil.getMqttPublishCommand(MQTTS, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials);
ArrayList<String> commands = new ArrayList<>();
@ -287,7 +288,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService
private String getDockerMqttPublishCommand(String protocol, String baseUrl, String deviceTelemetryTopic, DeviceCredentials deviceCredentials) throws URISyntaxException {
DeviceConnectivityInfo properties = getConnectivity(protocol);
String mqttHost = getHost(baseUrl, properties, protocol);
String mqttPort = properties.getPort().isEmpty() ? null : properties.getPort();
String mqttPort = getPort(properties);
return DeviceConnectivityUtil.getDockerMqttPublishCommand(protocol, baseUrl, mqttHost, mqttPort, deviceTelemetryTopic, deviceCredentials);
}
@ -327,14 +328,14 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService
private String getCoapPublishCommand(String protocol, String baseUrl, DeviceCredentials deviceCredentials) throws URISyntaxException {
DeviceConnectivityInfo properties = getConnectivity(protocol);
String hostName = getHost(baseUrl, properties, protocol);
String port = properties.getPort().isEmpty() ? "" : ":" + properties.getPort();
String port = StringUtils.isBlank(properties.getPort()) ? "" : ":" + properties.getPort();
return DeviceConnectivityUtil.getCoapPublishCommand(protocol, hostName, port, deviceCredentials);
}
private String getDockerCoapPublishCommand(String protocol, String baseUrl, DeviceCredentials deviceCredentials) throws URISyntaxException {
DeviceConnectivityInfo properties = getConnectivity(protocol);
String host = getHost(baseUrl, properties, protocol);
String port = properties.getPort().isEmpty() ? "" : ":" + properties.getPort();
String port = StringUtils.isBlank(properties.getPort()) ? "" : ":" + properties.getPort();
return DeviceConnectivityUtil.getDockerCoapPublishCommand(protocol, host, port, deviceCredentials);
}

2
dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java

@ -501,6 +501,8 @@ public class ModelConstants {
public static final String RESOURCE_ETAG_COLUMN = "etag";
public static final String RESOURCE_DESCRIPTOR_COLUMN = "descriptor";
public static final String RESOURCE_PREVIEW_COLUMN = "preview";
public static final String RESOURCE_IS_PUBLIC_COLUMN = "is_public";
public static final String PUBLIC_RESOURCE_KEY_COLUMN = "public_resource_key";
/**
* Ota Package constants.

12
dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceEntity.java

@ -37,8 +37,10 @@ import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_DATA_COLU
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_DESCRIPTOR_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_ETAG_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_IS_PUBLIC_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_PREVIEW_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.PUBLIC_RESOURCE_KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TENANT_ID_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TITLE_COLUMN;
@ -83,6 +85,12 @@ public class TbResourceEntity extends BaseSqlEntity<TbResource> {
@Column(name = RESOURCE_PREVIEW_COLUMN)
private byte[] preview;
@Column(name = RESOURCE_IS_PUBLIC_COLUMN)
private Boolean isPublic;
@Column(name = PUBLIC_RESOURCE_KEY_COLUMN, unique = true, updatable = false)
private String publicResourceKey;
@Column(name = EXTERNAL_ID_PROPERTY)
private UUID externalId;
@ -106,6 +114,8 @@ public class TbResourceEntity extends BaseSqlEntity<TbResource> {
this.etag = resource.getEtag();
this.descriptor = resource.getDescriptor();
this.preview = resource.getPreview();
this.isPublic = resource.isPublic();
this.publicResourceKey = resource.getPublicResourceKey();
this.externalId = getUuid(resource.getExternalId());
}
@ -123,6 +133,8 @@ public class TbResourceEntity extends BaseSqlEntity<TbResource> {
resource.setEtag(etag);
resource.setDescriptor(descriptor);
resource.setPreview(preview);
resource.setPublic(isPublic == null || isPublic);
resource.setPublicResourceKey(publicResourceKey);
resource.setExternalId(getEntityId(externalId, TbResourceId::new));
return resource;
}

12
dao/src/main/java/org/thingsboard/server/dao/model/sql/TbResourceInfoEntity.java

@ -37,7 +37,9 @@ import static org.thingsboard.server.dao.model.ModelConstants.EXTERNAL_ID_PROPER
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_DESCRIPTOR_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_ETAG_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_FILE_NAME_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_IS_PUBLIC_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.PUBLIC_RESOURCE_KEY_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TABLE_NAME;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TENANT_ID_COLUMN;
import static org.thingsboard.server.dao.model.ModelConstants.RESOURCE_TITLE_COLUMN;
@ -76,6 +78,12 @@ public class TbResourceInfoEntity extends BaseSqlEntity<TbResourceInfo> implemen
@Column(name = RESOURCE_DESCRIPTOR_COLUMN)
private JsonNode descriptor;
@Column(name = RESOURCE_IS_PUBLIC_COLUMN)
private Boolean isPublic;
@Column(name = PUBLIC_RESOURCE_KEY_COLUMN, unique = true, updatable = false)
private String publicResourceKey;
@Column(name = EXTERNAL_ID_PROPERTY)
private UUID externalId;
@ -95,6 +103,8 @@ public class TbResourceInfoEntity extends BaseSqlEntity<TbResourceInfo> implemen
this.etag = resource.getEtag();
this.fileName = resource.getFileName();
this.descriptor = resource.getDescriptor();
this.isPublic = resource.isPublic();
this.publicResourceKey = resource.getPublicResourceKey();
this.externalId = getUuid(resource.getExternalId());
}
@ -110,6 +120,8 @@ public class TbResourceInfoEntity extends BaseSqlEntity<TbResourceInfo> implemen
resource.setEtag(etag);
resource.setFileName(fileName);
resource.setDescriptor(descriptor);
resource.setPublic(isPublic == null || isPublic);
resource.setPublicResourceKey(publicResourceKey);
resource.setExternalId(getEntityId(externalId, TbResourceId::new));
return resource;
}

28
dao/src/main/java/org/thingsboard/server/dao/resource/BaseImageService.java

@ -23,6 +23,7 @@ import com.google.common.base.Strings;
import lombok.Data;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.commons.lang3.tuple.Pair;
@ -137,8 +138,8 @@ public class BaseImageService extends BaseResourceService implements ImageServic
@Override
@SneakyThrows
public TbResourceInfo saveImage(TbResource image) {
if (image.getId() == null && StringUtils.isEmpty(image.getResourceKey())) {
image.setResourceKey(getUniqueKey(image.getTenantId(), image.getFileName()));
if (image.getId() == null) {
image.setResourceKey(getUniqueKey(image.getTenantId(), StringUtils.defaultIfEmpty(image.getResourceKey(), image.getFileName())));
}
resourceValidator.validate(image, TbResourceInfo::getTenantId);
@ -149,6 +150,13 @@ public class BaseImageService extends BaseResourceService implements ImageServic
image.setDescriptorValue(descriptor);
image.setPreview(result.getRight());
if (StringUtils.isEmpty(image.getPublicResourceKey())) {
image.setPublicResourceKey(generatePublicResourceKey());
} else {
if (resourceInfoDao.existsByPublicResourceKey(ResourceType.IMAGE, image.getPublicResourceKey())) {
image.setPublicResourceKey(generatePublicResourceKey());
}
}
log.debug("[{}] Creating image {} ('{}')", image.getTenantId(), image.getResourceKey(), image.getName());
return new TbResourceInfo(doSaveResource(image));
}
@ -194,6 +202,10 @@ public class BaseImageService extends BaseResourceService implements ImageServic
return resourceKey;
}
private String generatePublicResourceKey() {
return RandomStringUtils.randomAlphanumeric(32);
}
@Override
public TbResourceInfo saveImageInfo(TbResourceInfo imageInfo) {
log.trace("Executing saveImageInfo [{}] [{}]", imageInfo.getTenantId(), imageInfo.getId());
@ -206,6 +218,11 @@ public class BaseImageService extends BaseResourceService implements ImageServic
return findResourceInfoByTenantIdAndKey(tenantId, ResourceType.IMAGE, key);
}
@Override
public TbResourceInfo getPublicImageInfoByKey(String publicResourceKey) {
return resourceInfoDao.findPublicResourceByKey(ResourceType.IMAGE, publicResourceKey);
}
@Override
public PageData<TbResourceInfo> getImagesByTenantId(TenantId tenantId, PageLink pageLink) {
log.trace("Executing getImagesByTenantId [{}]", tenantId);
@ -452,6 +469,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
image.setFileName(fileName);
image.setDescriptor(JacksonUtil.newObjectNode().put("mediaType", mdMediaType));
image.setData(imageData);
image.setPublic(true);
try {
imageInfo = saveImage(image);
} catch (Exception e) {
@ -602,7 +620,7 @@ public class BaseImageService extends BaseResourceService implements ImageServic
try {
ImageCacheKey key = getKeyFromUrl(tenantId, url);
if (key != null) {
var imageInfo = getImageInfoByTenantIdAndKey(key.getTenantId(), key.getKey());
var imageInfo = getImageInfoByTenantIdAndKey(key.getTenantId(), key.getResourceKey());
if (imageInfo != null) {
byte[] data = key.isPreview() ? getImagePreview(tenantId, imageInfo.getId()) : getImageData(tenantId, imageInfo.getId());
ImageDescriptor descriptor = getImageDescriptor(imageInfo, key.isPreview());
@ -638,9 +656,9 @@ public class BaseImageService extends BaseResourceService implements ImageServic
if (imageTenantId != null) {
var parts = url.split("/");
if (parts.length == 5) {
return new ImageCacheKey(imageTenantId, parts[4], false);
return ImageCacheKey.forImage(imageTenantId, parts[4], false);
} else if (parts.length == 6 && "preview".equals(parts[5])) {
return new ImageCacheKey(imageTenantId, parts[4], true);
return ImageCacheKey.forImage(imageTenantId, parts[4], true);
}
}
return null;

36
dao/src/main/java/org/thingsboard/server/dao/resource/ImageCacheKey.java

@ -15,14 +15,48 @@
*/
package org.thingsboard.server.dao.resource;
import lombok.AccessLevel;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import lombok.With;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.gen.transport.TransportProtos.ImageCacheKeyProto;
@Data
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public class ImageCacheKey {
private final TenantId tenantId;
private final String key;
private final String resourceKey;
@With
private final boolean preview;
private final String publicResourceKey;
public static ImageCacheKey forImage(TenantId tenantId, String key, boolean preview) {
return new ImageCacheKey(tenantId, key, preview, null);
}
public static ImageCacheKey forImage(TenantId tenantId, String key) {
return forImage(tenantId, key, false);
}
public static ImageCacheKey forPublicImage(String publicKey) {
return new ImageCacheKey(null, null, false, publicKey);
}
public ImageCacheKeyProto toProto() {
var msg = ImageCacheKeyProto.newBuilder();
if (resourceKey != null) {
msg.setResourceKey(resourceKey);
} else {
msg.setPublicResourceKey(publicResourceKey);
}
return msg.build();
}
public boolean isPublic() {
return this.publicResourceKey != null;
}
}

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save