Browse Source

Merge branch 'develop/3.4' into mapping-api-enpoints-swagger-fix

pull/6796/head
ivankozka 4 years ago
parent
commit
dcfdd7afc3
  1. 2
      application/src/main/conf/logback.xml
  2. 6
      application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java
  3. 2
      application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java
  4. 375
      application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java
  5. 10
      application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java
  6. 10
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java
  7. 30
      application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java
  8. 5
      application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java
  9. 31
      application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java
  10. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java
  11. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java
  12. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java
  13. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java
  14. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java
  15. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java
  16. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java
  17. 66
      application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java
  18. 11
      application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java
  19. 2
      application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java
  20. 2
      application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java
  21. 2
      application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java
  22. 2
      application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java
  23. 3
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java
  24. 18
      application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java
  25. 8
      application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java
  26. 2
      application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java
  27. 9
      application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java
  28. 2
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java
  29. 8
      application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java
  30. 11
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java
  31. 24
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java
  32. 8
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java
  33. 7
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java
  34. 5
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java
  35. 7
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java
  36. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java
  37. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java
  38. 6
      application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java
  39. 26
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java
  40. 143
      application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java
  41. 11
      application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java
  42. 9
      application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java
  43. 3
      application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java
  44. 13
      application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java
  45. 39
      application/src/main/resources/logback.xml
  46. 4
      application/src/main/resources/thingsboard.yml
  47. 2
      application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java
  48. 29
      application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java
  49. 20
      application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java
  50. 68
      application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java
  51. 118
      application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java
  52. 376
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java
  53. 56
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java
  54. 2
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java
  55. 116
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java
  56. 151
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java
  57. 7
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java
  58. 91
      application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java
  59. 11
      application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java
  60. 3
      application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java
  61. 78
      application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java
  62. 61
      application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java
  63. 214
      application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java
  64. 7
      application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java
  65. 4
      application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java
  66. 87
      application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java
  67. 50
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java
  68. 53
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java
  69. 1
      application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java
  70. 2
      common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java
  71. 43
      common/cluster-api/src/main/proto/queue.proto
  72. 1
      common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java
  73. 6
      common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java
  74. 39
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java
  75. 1
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java
  76. 5
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java
  77. 6
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java
  78. 1
      common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java
  79. 20
      common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java
  80. 7
      common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java
  81. 8
      common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java
  82. 74
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java
  83. 7
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java
  84. 31
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java
  85. 5
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java
  86. 13
      common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java
  87. 2
      dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java
  88. 2
      dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java
  89. 2
      dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java
  90. 18
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java
  91. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java
  92. 2
      dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java
  93. 6
      dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java
  94. 1
      dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java
  95. 18
      dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java
  96. 9
      dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java
  97. 134
      dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java
  98. 156
      dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java
  99. 3
      docker/.env
  100. 8
      docker/.gitignore

2
application/src/main/conf/logback.xml

@ -36,6 +36,8 @@
<logger name="org.thingsboard.server" level="INFO" />
<logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />
<logger name="org.apache.kafka.common.utils.AppInfoParser" level="WARN"/>
<logger name="org.apache.kafka.clients.consumer.ConsumerConfig" level="WARN"/>
<root level="INFO">
<appender-ref ref="fileLogAppender"/>

6
application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java

@ -80,6 +80,7 @@ import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.component.ComponentDiscoveryService;
import org.thingsboard.server.service.edge.rpc.EdgeRpcService;
import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.executors.ExternalCallExecutorService;
import org.thingsboard.server.service.executors.SharedEventLoopGroupService;
@ -220,6 +221,11 @@ public class ActorSystemContext {
@Getter
private EntityViewService entityViewService;
@Lazy
@Autowired(required = false)
@Getter
private TbEntityViewService tbEntityViewService;
@Autowired
@Getter
private TelemetrySubscriptionService tsSubService;

2
application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java

@ -137,6 +137,8 @@ public class ControllerConstants {
protected static final String EDGE_ASSIGN_RECEIVE_STEP_DESCRIPTION = "(Edge will receive this instantly, if it's currently connected, or once it's going to be connected to platform). ";
protected static final String ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the entity version name.";
protected static final String VERSION_ID_PARAM_DESCRIPTION = "Version id, for example fd82625bdd7d6131cf8027b44ee967012ecaf990. Represents commit hash.";
protected static final String BRANCH_PARAM_DESCRIPTION = "The name of the working branch, for example 'master'";
protected static final String MARKDOWN_CODE_BLOCK_START = "```json\n";
protected static final String MARKDOWN_CODE_BLOCK_END = "\n```";

375
application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java

@ -20,7 +20,6 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import lombok.Data;
import lombok.RequiredArgsConstructor;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.GetMapping;
@ -39,6 +38,7 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
@ -56,15 +56,24 @@ import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
import static org.thingsboard.server.controller.ControllerConstants.BRANCH_PARAM_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END;
import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START;
import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_DATA_PARAMETERS;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_SIZE_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.PAGE_NUMBER_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.SORT_PROPERTY_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.SORT_ORDER_ALLOWABLE_VALUES;
import static org.thingsboard.server.controller.ControllerConstants.TENANT_AUTHORITY_PARAGRAPH;
import static org.thingsboard.server.controller.ControllerConstants.VC_REQUEST_ID_PARAM_DESCRIPTION;
import static org.thingsboard.server.controller.ControllerConstants.VERSION_ID_PARAM_DESCRIPTION;
@RestController
@TbCoreComponent
@ -76,9 +85,20 @@ public class EntitiesVersionControlController extends BaseController {
private final EntitiesVersionControlService versionControlService;
@ApiOperation(value = "", notes = "" +
"SINGLE_ENTITY:" + NEW_LINE +
"```\n{\n" +
@ApiOperation(value = "Save entities version (saveEntitiesVersion)", notes = "" +
"Creates a new version of entities (or a single entity) by request.\n" +
"Supported entity types: CUSTOMER, ASSET, RULE_CHAIN, DASHBOARD, DEVICE_PROFILE, DEVICE, ENTITY_VIEW, WIDGETS_BUNDLE." + NEW_LINE +
"There are two available types of request: `SINGLE_ENTITY` and `COMPLEX`. " +
"Each of them contains version name (`versionName`) and name of a branch (`branch`) to create version (commit) in. " +
"If specified branch does not exists in a remote repo, then new empty branch will be created. " +
"Request of the `SINGLE_ENTITY` type has id of an entity (`entityId`) and additional configuration (`config`) " +
"which has following options: \n" +
"- `saveRelations` - whether to add inbound and outbound relations of type COMMON to created entity version;\n" +
"- `saveAttributes` - to save attributes of server scope (and also shared scope for devices);\n" +
"- `saveCredentials` - when saving a version of a device, to add its credentials to the version." + NEW_LINE +
"An example of a `SINGLE_ENTITY` version create request:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"type\": \"SINGLE_ENTITY\",\n" +
"\n" +
" \"versionName\": \"Version 1.0\",\n" +
@ -89,11 +109,25 @@ public class EntitiesVersionControlController extends BaseController {
" \"id\": \"b79448e0-d4f4-11ec-847b-0f432358ab48\"\n" +
" },\n" +
" \"config\": {\n" +
" \"saveRelations\": true\n" +
" \"saveRelations\": true,\n" +
" \"saveAttributes\": true,\n" +
" \"saveCredentials\": false\n" +
" }\n" +
"}\n```" + NEW_LINE +
"COMPLEX:" + NEW_LINE +
"```\n{\n" +
"}" +
MARKDOWN_CODE_BLOCK_END + NEW_LINE +
"Second request type (`COMPLEX`), additionally to `branch` and `versionName`, contains following properties:\n" +
"- `entityTypes` - a structure with entity types to export and configuration for each entity type; " +
" this configuration has all the options available for `SINGLE_ENTITY` and additionally has these ones: \n" +
" - `allEntities` and `entityIds` - if you want to save the version of all entities of the entity type " +
" then set `allEntities` param to true, otherwise set it to false and specify the list of specific entities (`entityIds`);\n" +
" - `syncStrategy` - synchronization strategy to use for this entity type: when set to `OVERWRITE` " +
" then the list of remote entities of this type will be overwritten by newly added entities. If set to " +
" `MERGE` - existing remote entities of this entity type will not be removed, new entities will just " +
" be added on top (or existing remote entities will be updated).\n" +
"- `syncStrategy` - default synchronization strategy to use when it is not specified for an entity type." + NEW_LINE +
"Example for this type of request:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"type\": \"COMPLEX\",\n" +
"\n" +
" \"versionName\": \"Devices and profiles: release 2\",\n" +
@ -104,7 +138,9 @@ public class EntitiesVersionControlController extends BaseController {
" \"DEVICE\": {\n" +
" \"syncStrategy\": null,\n" +
" \"allEntities\": true,\n" +
" \"saveRelations\": true\n" +
" \"saveRelations\": true,\n" +
" \"saveAttributes\": true,\n" +
" \"saveCredentials\": true\n" +
" },\n" +
" \"DEVICE_PROFILE\": {\n" +
" \"syncStrategy\": \"MERGE\",\n" +
@ -115,7 +151,11 @@ public class EntitiesVersionControlController extends BaseController {
" \"saveRelations\": true\n" +
" }\n" +
" }\n" +
"}\n```")
"}" +
MARKDOWN_CODE_BLOCK_END + NEW_LINE +
"Response wil contain generated request UUID, that can be then used to retrieve " +
"status of operation via `getVersionCreateRequestStatus`.\n" +
TENANT_AUTHORITY_PARAGRAPH)
@PostMapping("/version")
public DeferredResult<UUID> saveEntitiesVersion(@RequestBody VersionCreateRequest request) throws Exception {
SecurityUser user = getCurrentUser();
@ -123,7 +163,32 @@ public class EntitiesVersionControlController extends BaseController {
return wrapFuture(versionControlService.saveEntitiesVersion(user, request));
}
@ApiOperation(value = "", notes = "")
@ApiOperation(value = "Get version create request status (getVersionCreateRequestStatus)", notes = "" +
"Returns the status of previously made version create request. " + NEW_LINE +
"This status contains following properties:\n" +
"- `done` - whether request processing is finished;\n" +
"- `version` - created version info: timestamp, version id (commit hash), commit name and commit author;\n" +
"- `added` - count of items that were created in the remote repo;\n" +
"- `modified` - modified items count;\n" +
"- `removed` - removed items count;\n" +
"- `error` - error message, if an error occurred while handling the request." + NEW_LINE +
"An example of successful status:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"done\": true,\n" +
" \"added\": 10,\n" +
" \"modified\": 2,\n" +
" \"removed\": 5,\n" +
" \"version\": {\n" +
" \"timestamp\": 1655198528000,\n" +
" \"id\":\"8a834dd389ed80e0759ba8ee338b3f1fd160a114\",\n" +
" \"name\": \"My devices v2.0\",\n" +
" \"author\": \"John Doe\"\n" +
" },\n" +
" \"error\": null\n" +
"}" +
MARKDOWN_CODE_BLOCK_END +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{requestId}/status")
public VersionCreationResult getVersionCreateRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true)
@PathVariable UUID requestId) throws Exception {
@ -131,17 +196,49 @@ public class EntitiesVersionControlController extends BaseController {
return versionControlService.getVersionCreateStatus(getCurrentUser(), requestId);
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
" {\n" +
" \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" +
" \"name\": \"Device profile 1 version 1.0\"\n" +
" }\n" +
"]\n```")
@GetMapping(value = "/version/{branch}/{entityType}/{externalEntityUuid}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityVersions(@PathVariable String branch,
@ApiOperation(value = "List entity versions (listEntityVersions)", notes = "" +
"Returns list of versions for a specific entity in a concrete branch. \n" +
"You need to specify external id of an entity to list versions for. This is `externalId` property of an entity, " +
"or otherwise if not set - simply id of this entity. \n" +
"If specified branch does not exist - empty page data will be returned. " + NEW_LINE +
"Each version info item has timestamp, id, name and author. Version id can then be used to restore the version. " +
PAGE_DATA_PARAMETERS + NEW_LINE +
"Response example: \n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"data\": [\n" +
" {\n" +
" \"timestamp\": 1655198593000,\n" +
" \"id\": \"fd82625bdd7d6131cf8027b44ee967012ecaf990\",\n" +
" \"name\": \"Devices and assets - v2.0\",\n" +
" \"author\": \"John Doe <johndoe@gmail.com>\"\n" +
" },\n" +
" {\n" +
" \"timestamp\": 1655198528000,\n" +
" \"id\": \"682adcffa9c8a2f863af6f00c4850323acbd4219\",\n" +
" \"name\": \"Update my device\",\n" +
" \"author\": \"John Doe <johndoe@gmail.com>\"\n" +
" },\n" +
" {\n" +
" \"timestamp\": 1655198280000,\n" +
" \"id\": \"d2a6087c2b30e18cc55e7cdda345a8d0dfb959a4\",\n" +
" \"name\": \"Devices and assets - v1.0\",\n" +
" \"author\": \"John Doe <johndoe@gmail.com>\"\n" +
" }\n" +
" ],\n" +
" \"totalPages\": 1,\n" +
" \"totalElements\": 3,\n" +
" \"hasNext\": false\n" +
"}" +
MARKDOWN_CODE_BLOCK_END +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityVersions(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable EntityType entityType,
@ApiParam(value = "A string value representing external entity id. This is `externalId` property of an entity, or otherwise if not set - simply id of this entity.")
@PathVariable UUID externalEntityUuid,
@ApiParam(value = BRANCH_PARAM_DESCRIPTION)
@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -158,16 +255,17 @@ public class EntitiesVersionControlController extends BaseController {
return wrapFuture(versionControlService.listEntityVersions(getTenantId(), branch, externalEntityId, pageLink));
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
" {\n" +
" \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" +
" \"name\": \"Device profiles from dev\"\n" +
" }\n" +
"]\n```")
@GetMapping(value = "/version/{branch}/{entityType}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@PathVariable String branch,
@ApiOperation(value = "List entity type versions (listEntityTypeVersions)", notes = "" +
"Returns list of versions of an entity type in a branch. This is a collected list of versions that were created " +
"for entities of this type in a remote branch. \n" +
"If specified branch does not exist - empty page data will be returned. " +
"The response structure is the same as for `listEntityVersions` API method." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listEntityTypeVersions(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable EntityType entityType,
@ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true)
@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -183,23 +281,14 @@ public class EntitiesVersionControlController extends BaseController {
return wrapFuture(versionControlService.listEntityTypeVersions(getTenantId(), branch, entityType, pageLink));
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
" {\n" +
" \"id\": \"ba9baaca1742b730e7331f31a6a51da5fc7da7f7\",\n" +
" \"name\": \"Device 1 removed\"\n" +
" },\n" +
" {\n" +
" \"id\": \"b3c28d722d328324c7c15b0b30047b0c40011cf7\",\n" +
" \"name\": \"Device profiles added\"\n" +
" },\n" +
" {\n" +
" \"id\": \"c30c8bcaed3f0813649f0dee51a89d04d0a12b28\",\n" +
" \"name\": \"Devices added\"\n" +
" }\n" +
"]\n```")
@GetMapping(value = "/version/{branch}", params = {"pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listVersions(@PathVariable String branch,
@ApiOperation(value = "List all versions (listVersions)", notes = "" +
"Lists all available versions in a branch for all entity types. \n" +
"If specified branch does not exist - empty page data will be returned. " +
"The response format is the same as for `listEntityVersions` API method." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/version", params = {"branch", "pageSize", "page"})
public DeferredResult<PageData<EntityVersion>> listVersions(@ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true)
@RequestParam String branch,
@ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true)
@RequestParam int pageSize,
@ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true)
@ -216,43 +305,84 @@ public class EntitiesVersionControlController extends BaseController {
}
@GetMapping("/entity/{branch}/{entityType}/{versionId}")
public DeferredResult<List<VersionedEntityInfo>> listEntitiesAtVersion(@PathVariable String branch,
@ApiOperation(value = "List entities at version (listEntitiesAtVersion)", notes = "" +
"Returns a list of remote entities of a specific entity type that are available at a concrete version. \n" +
"Each entity item in the result has `externalId` property. " +
"Entities order will be the same as in the repository." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"})
public DeferredResult<List<VersionedEntityInfo>> listEntitiesAtVersion(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable EntityType entityType,
@PathVariable String versionId) throws Exception {
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@PathVariable String versionId,
@ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true)
@RequestParam String branch) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType));
}
@GetMapping("/entity/{branch}/{versionId}")
public DeferredResult<List<VersionedEntityInfo>> listAllEntitiesAtVersion(@PathVariable String branch,
@PathVariable String versionId) throws Exception {
@ApiOperation(value = "List all entities at version (listAllEntitiesAtVersion)", notes = "" +
"Returns a list of all remote entities available in a specific version. " +
"Response type is the same as for listAllEntitiesAtVersion API method. \n" +
"Returned entities order will be the same as in the repository." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/entity/{versionId}", params = {"branch"})
public DeferredResult<List<VersionedEntityInfo>> listAllEntitiesAtVersion(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@PathVariable String versionId,
@ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true)
@RequestParam String branch) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId));
}
@ApiOperation(value = "Get entity data info (getEntityDataInfo)", notes = "" +
"Retrieves short info about the remote entity by external id at a concrete version. \n" +
"Returned entity data info contains following properties: " +
"`hasRelations` (whether stored entity data contains relations), `hasAttributes` (contains attributes) and " +
"`hasCredentials` (whether stored device data has credentials)." +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping("/info/{versionId}/{entityType}/{externalEntityUuid}")
public DeferredResult<EntityDataInfo> getEntityDataInfo(@PathVariable String versionId,
public DeferredResult<EntityDataInfo> getEntityDataInfo(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@PathVariable String versionId,
@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable EntityType entityType,
@ApiParam(value = "A string value representing external entity id", required = true)
@PathVariable UUID externalEntityUuid) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid);
return wrapFuture(versionControlService.getEntityDataInfo(getCurrentUser(), entityId, versionId));
}
@GetMapping("/diff/{branch}/{entityType}/{internalEntityUuid}")
public DeferredResult<EntityDataDiff> compareEntityDataToVersion(@PathVariable String branch,
@ApiOperation(value = "Compare entity data to version (compareEntityDataToVersion)", notes = "" +
"Returns an object with current entity data and the one at a specific version. " +
"Entity data structure is the same as stored in a repository. " +
TENANT_AUTHORITY_PARAGRAPH)
@GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"})
public DeferredResult<EntityDataDiff> compareEntityDataToVersion(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true)
@PathVariable EntityType entityType,
@ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true)
@PathVariable UUID internalEntityUuid,
@ApiParam(value = BRANCH_PARAM_DESCRIPTION)
@RequestParam String branch,
@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true)
@RequestParam String versionId) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, internalEntityUuid);
return wrapFuture(versionControlService.compareEntityDataToVersion(getCurrentUser(), branch, entityId, versionId));
}
@ApiOperation(value = "", notes = "" +
"SINGLE_ENTITY:" + NEW_LINE +
"```\n{\n" +
@ApiOperation(value = "Load entities version (loadEntitiesVersion)", notes = "" +
"Loads specific version of remote entities (or single entity) by request. " +
"Supported entity types: CUSTOMER, ASSET, RULE_CHAIN, DASHBOARD, DEVICE_PROFILE, DEVICE, ENTITY_VIEW, WIDGETS_BUNDLE." + NEW_LINE +
"There are multiple types of request. Each of them requires branch name (`branch`) and version id (`versionId`). " +
"Request of type `SINGLE_ENTITY` is needed to restore a concrete version of a specific entity. It contains " +
"id of a remote entity (`externalEntityId`) and additional configuration (`config`):\n" +
"- `loadRelations` - to update relations list (in case `saveRelations` option was enabled during version creation);\n" +
"- `loadAttributes` - to load entity attributes (if `saveAttributes` config option was enabled);\n" +
"- `loadCredentials` - to update device credentials (if `saveCredentials` option was enabled during version creation)." + NEW_LINE +
"An example of such request:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"type\": \"SINGLE_ENTITY\",\n" +
" \n" +
" \"branch\": \"dev\",\n" +
@ -264,11 +394,23 @@ public class EntitiesVersionControlController extends BaseController {
" },\n" +
" \"config\": {\n" +
" \"loadRelations\": false,\n" +
" \"findExistingEntityByName\": false\n" +
" \"loadAttributes\": true,\n" +
" \"loadCredentials\": true\n" +
" }\n" +
"}\n```" + NEW_LINE +
"ENTITY_TYPE:" + NEW_LINE +
"```\n{\n" +
"}" +
MARKDOWN_CODE_BLOCK_END + NEW_LINE +
"Another request type (`ENTITY_TYPE`) is needed to load specific version of the whole entity types. " +
"It contains a structure with entity types to load and configs for each entity type (`entityTypes`). " +
"For each specified entity type, the method will load all remote entities of this type that are present " +
"at the version. A config for each entity type contains the same options as in `SINGLE_ENTITY` request type, and " +
"additionally contains following options:\n" +
"- `removeOtherEntities` - to remove local entities that are not present on the remote - basically to " +
" overwrite local entity type with the remote one;\n" +
"- `findExistingEntityByName` - when you are loading some remote entities that are not yet present at this tenant, " +
" try to find existing entity by name and update it rather than create new." + NEW_LINE +
"Here is an example of the request to completely restore version of the whole device entity type:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"type\": \"ENTITY_TYPE\",\n" +
"\n" +
" \"branch\": \"dev\",\n" +
@ -276,12 +418,18 @@ public class EntitiesVersionControlController extends BaseController {
"\n" +
" \"entityTypes\": {\n" +
" \"DEVICE\": {\n" +
" \"loadRelations\": false,\n" +
" \"removeOtherEntities\": true,\n" +
" \"findExistingEntityByName\": false,\n" +
" \"removeOtherEntities\": true\n" +
" \"loadRelations\": true,\n" +
" \"loadAttributes\": true,\n" +
" \"loadCredentials\": true\n" +
" }\n" +
" }\n" +
"}\n```")
"}" +
MARKDOWN_CODE_BLOCK_END + NEW_LINE +
"The response will contain generated request UUID that is to be used to check the status of operation " +
"via `getVersionLoadRequestStatus`." +
TENANT_AUTHORITY_PARAGRAPH)
@PostMapping("/entity")
public UUID loadEntitiesVersion(@RequestBody VersionLoadRequest request) throws Exception {
SecurityUser user = getCurrentUser();
@ -289,17 +437,54 @@ public class EntitiesVersionControlController extends BaseController {
return versionControlService.loadEntitiesVersion(user, request);
}
@ApiOperation(value = "", notes = "")
@ApiOperation(value = "Get version load request status (getVersionLoadRequestStatus)", notes = "" +
"Returns the status of previously made version load request. " +
"The structure contains following parameters:\n" +
"- `done` - if the request was successfully processed;\n" +
"- `result` - a list of load results for each entity type:\n" +
" - `created` - created entities count;\n" +
" - `updated` - updated entities count;\n" +
" - `deleted` - removed entities count.\n" +
"- `error` - if an error occurred during processing, error info:\n" +
" - `type` - error type;\n" +
" - `source` - an external id of remote entity;\n" +
" - `target` - if failed to find referenced entity by external id - this external id;\n" +
" - `message` - error message." + NEW_LINE +
"An example of successfully processed request status:\n" +
MARKDOWN_CODE_BLOCK_START +
"{\n" +
" \"done\": true,\n" +
" \"result\": [\n" +
" {\n" +
" \"entityType\": \"DEVICE\",\n" +
" \"created\": 10,\n" +
" \"updated\": 5,\n" +
" \"deleted\": 5\n" +
" },\n" +
" {\n" +
" \"entityType\": \"ASSET\",\n" +
" \"created\": 4,\n" +
" \"updated\": 0,\n" +
" \"deleted\": 8\n" +
" }\n" +
" ]\n" +
"}" +
MARKDOWN_CODE_BLOCK_END +
TENANT_AUTHORITY_PARAGRAPH
)
@GetMapping(value = "/entity/{requestId}/status")
public VersionLoadResult getVersionLoadRequestStatus(@ApiParam(value = VC_REQUEST_ID_PARAM_DESCRIPTION, required = true)
@PathVariable UUID requestId) throws Exception {
@PathVariable UUID requestId) throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
return versionControlService.getVersionLoadStatus(getCurrentUser(), requestId);
}
@ApiOperation(value = "", notes = "" +
"```\n[\n" +
@ApiOperation(value = "List branches (listBranches)", notes = "" +
"Lists branches available in the remote repository. \n\n" +
"Response example: \n" +
MARKDOWN_CODE_BLOCK_START +
"[\n" +
" {\n" +
" \"name\": \"master\",\n" +
" \"default\": true\n" +
@ -312,37 +497,29 @@ public class EntitiesVersionControlController extends BaseController {
" \"name\": \"dev-2\",\n" +
" \"default\": false\n" +
" }\n" +
"]\n\n```")
"]" +
MARKDOWN_CODE_BLOCK_END)
@GetMapping("/branches")
public DeferredResult<List<BranchInfo>> listBranches() throws ThingsboardException {
try {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
final TenantId tenantId = getTenantId();
ListenableFuture<List<String>> branches = versionControlService.listBranches(tenantId);
return wrapFuture(Futures.transform(branches, remoteBranches -> {
List<BranchInfo> infos = new ArrayList<>();
String defaultBranch = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch();
if (StringUtils.isNotEmpty(defaultBranch)) {
infos.add(new BranchInfo(defaultBranch, true));
}
remoteBranches.forEach(branch -> {
if (!branch.equals(defaultBranch)) {
infos.add(new BranchInfo(branch, false));
}
});
return infos;
}, MoreExecutors.directExecutor()));
} catch (Exception e) {
throw handleException(e);
}
}
@Data
public static class BranchInfo {
private final String name;
private final boolean isDefault;
public DeferredResult<List<BranchInfo>> listBranches() throws Exception {
accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ);
final TenantId tenantId = getTenantId();
ListenableFuture<List<BranchInfo>> branches = versionControlService.listBranches(tenantId);
return wrapFuture(Futures.transform(branches, remoteBranches -> {
List<BranchInfo> infos = new ArrayList<>();
BranchInfo defaultBranch;
String defaultBranchName = versionControlService.getVersionControlSettings(tenantId).getDefaultBranch();
if (StringUtils.isNotEmpty(defaultBranchName)) {
defaultBranch = new BranchInfo(defaultBranchName, true);
} else {
defaultBranch = remoteBranches.stream().filter(BranchInfo::isDefault).findFirst().orElse(null);
}
if (defaultBranch != null) {
infos.add(defaultBranch);
}
infos.addAll(remoteBranches.stream().filter(b -> !b.equals(defaultBranch))
.map(b -> new BranchInfo(b.getName(), false)).collect(Collectors.toList()));
return infos;
}, MoreExecutors.directExecutor()));
}
}

10
application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java

@ -161,20 +161,20 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe
configurationDescriptor.set("nodeDefinition", node);
scannedComponent.setConfigurationDescriptor(configurationDescriptor);
scannedComponent.setClazz(clazzName);
log.info("Processing scanned component: {}", scannedComponent);
log.debug("Processing scanned component: {}", scannedComponent);
} catch (Exception e) {
log.error("Can't initialize component {}, due to {}", def.getBeanClassName(), e.getMessage(), e);
throw new RuntimeException(e);
}
ComponentDescriptor persistedComponent = componentDescriptorService.findByClazz(TenantId.SYS_TENANT_ID, clazzName);
if (persistedComponent == null) {
log.info("Persisting new component: {}", scannedComponent);
log.debug("Persisting new component: {}", scannedComponent);
scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent);
} else if (scannedComponent.equals(persistedComponent)) {
log.info("Component is already persisted: {}", persistedComponent);
log.debug("Component is already persisted: {}", persistedComponent);
scannedComponent = persistedComponent;
} else {
log.info("Component {} will be updated to {}", persistedComponent, scannedComponent);
log.debug("Component {} will be updated to {}", persistedComponent, scannedComponent);
componentDescriptorService.deleteByClazz(TenantId.SYS_TENANT_ID, persistedComponent.getClazz());
scannedComponent.setId(persistedComponent.getId());
scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent);
@ -224,7 +224,7 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe
@Override
public void discoverComponents() {
registerRuleNodeComponents();
log.info("Found following definitions: {}", components.values());
log.debug("Found following definitions: {}", components.values());
}
@Override

10
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java

@ -21,6 +21,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EdgeUtils;
@ -53,6 +54,8 @@ import org.thingsboard.server.dao.service.DataValidator;
import org.thingsboard.server.dao.user.UserService;
import org.thingsboard.server.dao.widget.WidgetTypeService;
import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.provider.TbQueueProducerProvider;
import org.thingsboard.server.service.edge.rpc.constructor.AdminSettingsMsgConstructor;
import org.thingsboard.server.service.edge.rpc.constructor.AlarmMsgConstructor;
import org.thingsboard.server.service.edge.rpc.constructor.AssetMsgConstructor;
@ -142,6 +145,13 @@ public abstract class BaseEdgeProcessor {
@Autowired
protected OtaPackageService otaPackageService;
@Autowired
protected PartitionService partitionService;
@Autowired
@Lazy
protected TbQueueProducerProvider producerProvider;
@Autowired
protected DataValidator<Device> deviceValidator;

30
application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/TelemetryEdgeProcessor.java

@ -52,6 +52,8 @@ import org.thingsboard.server.common.data.kv.AttributeKey;
import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.msg.TbMsg;
import org.thingsboard.server.common.msg.TbMsgMetaData;
import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.common.msg.session.SessionMsgType;
import org.thingsboard.server.common.transport.adaptor.JsonConverter;
import org.thingsboard.server.common.transport.util.JsonUtils;
@ -61,9 +63,12 @@ import org.thingsboard.server.gen.edge.v1.EntityDataProto;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.TbQueueCallback;
import org.thingsboard.server.queue.TbQueueMsgMetadata;
import org.thingsboard.server.queue.TbQueueProducer;
import org.thingsboard.server.queue.common.TbProtoQueueMsg;
import org.thingsboard.server.queue.util.TbCoreComponent;
import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@ -77,14 +82,19 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor {
private final Gson gson = new Gson();
private TbQueueProducer<TbProtoQueueMsg<TransportProtos.ToCoreMsg>> tbCoreMsgProducer;
@PostConstruct
public void init() {
tbCoreMsgProducer = producerProvider.getTbCoreMsgProducer();
}
public List<ListenableFuture<Void>> processTelemetryFromEdge(TenantId tenantId, CustomerId customerId, EntityDataProto entityData) {
log.trace("[{}] onTelemetryUpdate [{}]", tenantId, entityData);
List<ListenableFuture<Void>> result = new ArrayList<>();
EntityId entityId = constructEntityId(entityData);
if ((entityData.hasPostAttributesMsg() || entityData.hasPostTelemetryMsg() || entityData.hasAttributesUpdatedMsg()) && entityId != null) {
// @voba - in terms of performance we should not fetch device from DB by id
// TbMsgMetaData metaData = constructBaseMsgMetadata(tenantId, entityId);
TbMsgMetaData metaData = new TbMsgMetaData();
TbMsgMetaData metaData = constructBaseMsgMetadata(tenantId, entityId);
metaData.putValue(DataConstants.MSG_SOURCE_KEY, DataConstants.EDGE_MSG_SOURCE);
if (entityData.hasPostAttributesMsg()) {
result.add(processPostAttributes(tenantId, customerId, entityId, entityData.getPostAttributesMsg(), metaData));
@ -96,6 +106,20 @@ public class TelemetryEdgeProcessor extends BaseEdgeProcessor {
if (entityData.hasPostTelemetryMsg()) {
result.add(processPostTelemetry(tenantId, customerId, entityId, entityData.getPostTelemetryMsg(), metaData));
}
if (EntityType.DEVICE.equals(entityId.getEntityType())) {
DeviceId deviceId = new DeviceId(entityId.getId());
TransportProtos.DeviceActivityProto deviceActivityMsg = TransportProtos.DeviceActivityProto.newBuilder()
.setTenantIdMSB(tenantId.getId().getMostSignificantBits())
.setTenantIdLSB(tenantId.getId().getLeastSignificantBits())
.setDeviceIdMSB(deviceId.getId().getMostSignificantBits())
.setDeviceIdLSB(deviceId.getId().getLeastSignificantBits())
.setLastActivityTime(System.currentTimeMillis()).build();
TopicPartitionInfo tpi = partitionService.resolve(ServiceType.TB_CORE, tenantId, deviceId);
tbCoreMsgProducer.send(tpi, new TbProtoQueueMsg<>(deviceId.getId(),
TransportProtos.ToCoreMsg.newBuilder().setDeviceActivityMsg(deviceActivityMsg).build()), null);
}
}
if (entityData.hasAttributeDeleteMsg()) {
result.add(processAttributeDeleteMsg(tenantId, entityId, entityData.getAttributeDeleteMsg(), entityData.getEntityType()));

5
application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java

@ -25,6 +25,7 @@ import com.google.common.util.concurrent.SettableFuture;
import lombok.extern.slf4j.Slf4j;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Device;
@ -72,6 +73,7 @@ import org.thingsboard.server.gen.edge.v1.RelationRequestMsg;
import org.thingsboard.server.gen.edge.v1.RuleChainMetadataRequestMsg;
import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg;
import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg;
import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import java.util.ArrayList;
@ -100,8 +102,9 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService {
@Autowired
private DeviceService deviceService;
@Lazy
@Autowired
private EntityViewService entityViewService;
private TbEntityViewService entityViewService;
@Autowired
private DeviceProfileService deviceProfileService;

31
application/src/main/java/org/thingsboard/server/service/entitiy/AbstractTbEntityService.java

@ -21,6 +21,7 @@ import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.HasName;
@ -64,6 +65,7 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.edge.EdgeNotificationService;
import org.thingsboard.server.service.executors.DbCallbackExecutorService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.EntitiesVersionControlService;
import org.thingsboard.server.service.install.InstallScripts;
import org.thingsboard.server.service.ota.OtaPackageStateService;
@ -77,6 +79,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
@Slf4j
@ -93,13 +96,13 @@ public abstract class AbstractTbEntityService {
@Autowired
protected DbCallbackExecutorService dbExecutor;
@Autowired
@Autowired(required = false)
protected TbNotificationEntityService notificationEntityService;
@Autowired(required = false)
protected EdgeService edgeService;
@Autowired
protected AlarmService alarmService;
@Autowired
@Autowired(required = false)
protected EntityActionService entityActionService;
@Autowired
protected DeviceService deviceService;
@ -111,24 +114,27 @@ public abstract class AbstractTbEntityService {
protected TenantService tenantService;
@Autowired
protected CustomerService customerService;
@Autowired
@Lazy
@Autowired(required = false)
protected ClaimDevicesService claimDevicesService;
@Autowired
protected TbTenantProfileCache tenantProfileCache;
@Autowired
protected RuleChainService ruleChainService;
@Autowired
@Autowired(required = false)
protected TbRuleChainService tbRuleChainService;
@Autowired
@Autowired(required = false)
protected EdgeNotificationService edgeNotificationService;
@Autowired
protected QueueService queueService;
@Autowired
protected DashboardService dashboardService;
@Autowired
protected EntitiesVersionControlService vcService;
@Autowired(required = false)
private EntitiesVersionControlService vcService;
@Autowired
protected EntityViewService entityViewService;
@Lazy
@Autowired
protected TelemetrySubscriptionService tsSubService;
@Autowired
@ -149,7 +155,7 @@ public abstract class AbstractTbEntityService {
protected InstallScripts installScripts;
@Autowired
protected UserService userService;
@Autowired
@Autowired(required = false)
protected TbResourceService resourceService;
@Autowired
protected WidgetsBundleService widgetsBundleService;
@ -245,4 +251,13 @@ public abstract class AbstractTbEntityService {
}
return result;
}
protected ListenableFuture<UUID> autoCommit(SecurityUser user, EntityId entityId) throws Exception {
if (vcService != null) {
return vcService.autoCommit(user, entityId);
} else {
// We do not support auto-commit for rule engine
return Futures.immediateFailedFuture(new RuntimeException("Operation not supported!"));
}
}
}

2
application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java

@ -128,7 +128,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS
}
@Override
public void notifyCreateOruUpdateTenant(Tenant tenant, ComponentLifecycleEvent event) {
public void notifyCreateOrUpdateTenant(Tenant tenant, ComponentLifecycleEvent event) {
tbClusterService.onTenantChange(tenant, null);
tbClusterService.broadcastEntityStateChangeEvent(tenant.getId(), tenant.getId(), event);
}

2
application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java

@ -71,7 +71,7 @@ public interface TbNotificationEntityService {
E entity, ActionType actionType,
SecurityUser user, Object... additionalInfo);
void notifyCreateOruUpdateTenant(Tenant tenant, ComponentLifecycleEvent event);
void notifyCreateOrUpdateTenant(Tenant tenant, ComponentLifecycleEvent event);
void notifyDeleteTenant(Tenant tenant);

2
application/src/main/java/org/thingsboard/server/service/entitiy/asset/DefaultTbAssetService.java

@ -47,7 +47,7 @@ public class DefaultTbAssetService extends AbstractTbEntityService implements Tb
TenantId tenantId = asset.getTenantId();
try {
Asset savedAsset = checkNotNull(assetService.saveAsset(asset));
vcService.autoCommit(user, savedAsset.getId());
autoCommit(user, savedAsset.getId());
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedAsset.getId(), savedAsset, savedAsset.getCustomerId(), actionType, user);
return savedAsset;
} catch (Exception e) {

2
application/src/main/java/org/thingsboard/server/service/entitiy/customer/DefaultTbCustomerService.java

@ -42,7 +42,7 @@ public class DefaultTbCustomerService extends AbstractTbEntityService implements
TenantId tenantId = customer.getTenantId();
try {
Customer savedCustomer = checkNotNull(customerService.saveCustomer(customer));
vcService.autoCommit(user, savedCustomer.getId());
autoCommit(user, savedCustomer.getId());
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedCustomer.getId(), savedCustomer, null, actionType, user);
return savedCustomer;
} catch (Exception e) {

2
application/src/main/java/org/thingsboard/server/service/entitiy/dashboard/DefaultTbDashboardService.java

@ -48,7 +48,7 @@ public class DefaultTbDashboardService extends AbstractTbEntityService implement
TenantId tenantId = dashboard.getTenantId();
try {
Dashboard savedDashboard = checkNotNull(dashboardService.saveDashboard(dashboard));
vcService.autoCommit(user, savedDashboard.getId());
autoCommit(user, savedDashboard.getId());
notificationEntityService.notifyCreateOrUpdateEntity(tenantId, savedDashboard.getId(), savedDashboard,
null, actionType, user);
return savedDashboard;

2
application/src/main/java/org/thingsboard/server/service/entitiy/device/DefaultTbDeviceService.java

@ -54,7 +54,7 @@ public class DefaultTbDeviceService extends AbstractTbEntityService implements T
ActionType actionType = device.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
try {
Device savedDevice = checkNotNull(deviceService.saveDeviceWithAccessToken(device, accessToken));
vcService.autoCommit(user, savedDevice.getId());
autoCommit(user, savedDevice.getId());
notificationEntityService.notifyCreateOrUpdateDevice(tenantId, savedDevice.getId(), savedDevice.getCustomerId(),
savedDevice, oldDevice, actionType, user);

2
application/src/main/java/org/thingsboard/server/service/entitiy/deviceProfile/DefaultTbDeviceProfileService.java

@ -54,7 +54,7 @@ public class DefaultTbDeviceProfileService extends AbstractTbEntityService imple
}
}
DeviceProfile savedDeviceProfile = checkNotNull(deviceProfileService.saveDeviceProfile(deviceProfile));
vcService.autoCommit(user, savedDeviceProfile.getId());
autoCommit(user, savedDeviceProfile.getId());
tbClusterService.onDeviceProfileChange(savedDeviceProfile, null);
tbClusterService.broadcastEntityStateChangeEvent(tenantId, savedDeviceProfile.getId(),
actionType.equals(ActionType.ADDED) ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);

66
application/src/main/java/org/thingsboard/server/service/entitiy/entityView/DefaultTbEntityViewService.java

@ -23,6 +23,7 @@ import com.google.common.util.concurrent.SettableFuture;
import lombok.AllArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.springframework.util.ConcurrentReferenceHashMap;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.DataConstants;
import org.thingsboard.server.common.data.EntityType;
@ -40,8 +41,9 @@ import org.thingsboard.server.common.data.kv.AttributeKvEntry;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.entitiy.AbstractTbEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
@ -50,19 +52,22 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.stream.Collectors;
import static org.apache.commons.lang3.StringUtils.isBlank;
@Service
@TbCoreComponent
@AllArgsConstructor
@Slf4j
public class DefaultTbEntityViewService extends AbstractTbEntityService implements TbEntityViewService {
private final TimeseriesService tsService;
final Map<TenantId, Map<EntityId, List<EntityView>>> localCache = new ConcurrentHashMap<>();
@Override
public EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException {
ActionType actionType = entityView.getId() == null ? ActionType.ADDED : ActionType.UPDATED;
@ -71,6 +76,9 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
this.updateEntityViewAttributes(user, savedEntityView, existingEntityView);
notificationEntityService.notifyCreateOrUpdateEntity(savedEntityView.getTenantId(), savedEntityView.getId(), savedEntityView,
null, actionType, user);
localCache.computeIfAbsent(savedEntityView.getTenantId(), (k) -> new ConcurrentReferenceHashMap<>()).clear();
tbClusterService.broadcastEntityStateChangeEvent(savedEntityView.getTenantId(), savedEntityView.getId(),
entityView.getId() == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
return savedEntityView;
} catch (Exception e) {
notificationEntityService.notifyEntity(user.getTenantId(), emptyId(EntityType.ENTITY_VIEW), entityView, null, actionType, user, e);
@ -122,6 +130,9 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
entityViewService.deleteEntityView(tenantId, entityViewId);
notificationEntityService.notifyDeleteEntity(tenantId, entityViewId, entityView, entityView.getCustomerId(), ActionType.DELETED,
relatedEdgeIds, user, entityViewId.toString());
localCache.computeIfAbsent(tenantId, (k) -> new ConcurrentReferenceHashMap<>()).clear();
tbClusterService.broadcastEntityStateChangeEvent(tenantId, entityViewId, ComponentLifecycleEvent.DELETED);
} catch (Exception e) {
notificationEntityService.notifyEntity(tenantId, emptyId(EntityType.ENTITY_VIEW), null, null,
ActionType.DELETED, user, e, entityViewId.toString());
@ -214,6 +225,51 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
}
}
@Override
public ListenableFuture<List<EntityView>> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId) {
Map<EntityId, List<EntityView>> localCacheByTenant = localCache.computeIfAbsent(tenantId, (k) -> new ConcurrentReferenceHashMap<>());
List<EntityView> fromLocalCache = localCacheByTenant.get(entityId);
if (fromLocalCache != null) {
return Futures.immediateFuture(fromLocalCache);
}
ListenableFuture<List<EntityView>> future = entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId);
return Futures.transform(future, (entityViewList) -> {
localCacheByTenant.put(entityId, entityViewList);
return entityViewList;
}, MoreExecutors.directExecutor());
}
@Override
public void onComponentLifecycleMsg(ComponentLifecycleMsg componentLifecycleMsg) {
Map<EntityId, List<EntityView>> localCacheByTenant = localCache.computeIfAbsent(componentLifecycleMsg.getTenantId(), (k) -> new ConcurrentReferenceHashMap<>());
EntityViewId entityViewId = new EntityViewId(componentLifecycleMsg.getEntityId().getId());
deleteOldCacheValue(localCacheByTenant, entityViewId);
if (componentLifecycleMsg.getEvent() != ComponentLifecycleEvent.DELETED) {
EntityView entityView = entityViewService.findEntityViewById(componentLifecycleMsg.getTenantId(), entityViewId);
if (entityView != null) {
localCacheByTenant.remove(entityView.getEntityId());
}
}
}
private void deleteOldCacheValue(Map<EntityId, List<EntityView>> localCacheByTenant, EntityViewId entityViewId) {
for (var entry : localCacheByTenant.entrySet()) {
EntityView toDelete = null;
for (EntityView view : entry.getValue()) {
if (entityViewId.equals(view.getId())) {
toDelete = view;
break;
}
}
if (toDelete != null) {
entry.getValue().remove(toDelete);
break;
}
}
}
private ListenableFuture<List<Void>> copyAttributesFromEntityToEntityView(EntityView entityView, String scope, Collection<String> keys, SecurityUser user) throws ThingsboardException {
EntityViewId entityId = entityView.getId();
if (keys != null && !keys.isEmpty()) {
@ -229,8 +285,8 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
long lastUpdateTs = attributeKvEntry.getLastUpdateTs();
return startTime == 0 && endTime == 0 ||
(endTime == 0 && startTime < lastUpdateTs) ||
(startTime == 0 && endTime > lastUpdateTs)
? true : startTime < lastUpdateTs && endTime > lastUpdateTs;
(startTime == 0 && endTime > lastUpdateTs) ||
(startTime < lastUpdateTs && endTime > lastUpdateTs);
}).collect(Collectors.toList());
tsSubService.saveAndNotify(entityView.getTenantId(), entityId, scope, attributes, new FutureCallback<Void>() {
@Override
@ -345,7 +401,7 @@ public class DefaultTbEntityViewService extends AbstractTbEntityService implemen
@Override
public void onFailure(Throwable t) {
try {
logTimeseriesDeleted(entityView.getTenantId(),user, entityId, keys, t);
logTimeseriesDeleted(entityView.getTenantId(), user, entityId, keys, t);
} catch (ThingsboardException e) {
log.error("Failed to log timeseries delete", e);
}

11
application/src/main/java/org/thingsboard/server/service/entitiy/entityView/TbEntityViewService.java

@ -15,22 +15,27 @@
*/
package org.thingsboard.server.service.entitiy.entityView;
import com.google.common.util.concurrent.ListenableFuture;
import org.thingsboard.server.common.data.Customer;
import org.thingsboard.server.common.data.EntityView;
import org.thingsboard.server.common.data.edge.Edge;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.CustomerId;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.msg.plugin.ComponentLifecycleListener;
import org.thingsboard.server.service.security.model.SecurityUser;
public interface TbEntityViewService {
import java.util.List;
public interface TbEntityViewService extends ComponentLifecycleListener {
EntityView save(EntityView entityView, EntityView existingEntityView, SecurityUser user) throws ThingsboardException;
void updateEntityViewAttributes(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException;
void delete (EntityView entity, SecurityUser user) throws ThingsboardException;
void delete (EntityView entity, SecurityUser user) throws ThingsboardException;
EntityView assignEntityViewToCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer,
SecurityUser user) throws ThingsboardException;
@ -46,4 +51,6 @@ public interface TbEntityViewService {
EntityView unassignEntityViewFromCustomer(TenantId tenantId, EntityViewId entityViewId, Customer customer,
SecurityUser user) throws ThingsboardException;
ListenableFuture<List<EntityView>> findEntityViewsByTenantIdAndEntityIdAsync(TenantId tenantId, EntityId entityId);
}

2
application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java

@ -54,7 +54,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T
installScripts.createDefaultEdgeRuleChains(savedTenant.getId());
}
tenantProfileCache.evict(savedTenant.getId());
notificationEntityService.notifyCreateOruUpdateTenant(savedTenant, created ?
notificationEntityService.notifyCreateOrUpdateTenant(savedTenant, created ?
ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
TenantProfile oldTenantProfile = oldTenant != null ? tenantProfileService.findTenantProfileById(TenantId.SYS_TENANT_ID, oldTenant.getTenantProfileId()) : null;

2
application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java

@ -25,6 +25,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Profile;
import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder;
import org.springframework.stereotype.Service;
@ -162,6 +163,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService {
@Getter
private boolean persistActivityToTelemetry;
@Lazy
@Autowired
private QueueService queueService;

2
application/src/main/java/org/thingsboard/server/service/install/SqlDatabaseUpgradeService.java

@ -21,6 +21,7 @@ import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntitySubtype;
@ -121,6 +122,7 @@ public class SqlDatabaseUpgradeService implements DatabaseEntitiesUpgradeService
@Autowired
private ApiUsageStateService apiUsageStateService;
@Lazy
@Autowired
private QueueService queueService;

2
application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java

@ -22,6 +22,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.JacksonUtil;
@ -117,6 +118,7 @@ public class DefaultDataUpdateService implements DataUpdateService {
@Autowired
private TenantProfileService tenantProfileService;
@Lazy
@Autowired
private QueueService queueService;

3
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java

@ -364,13 +364,14 @@ public class DefaultTbClusterService implements TbClusterService {
private void broadcast(ComponentLifecycleMsg msg) {
byte[] msgBytes = encodingService.encode(msg);
TbQueueProducer<TbProtoQueueMsg<ToRuleEngineNotificationMsg>> toRuleEngineProducer = producerProvider.getRuleEngineNotificationsMsgProducer();
Set<String> tbRuleEngineServices = new HashSet<>(partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE));
Set<String> tbRuleEngineServices = partitionService.getAllServiceIds(ServiceType.TB_RULE_ENGINE);
EntityType entityType = msg.getEntityId().getEntityType();
if (entityType.equals(EntityType.TENANT)
|| entityType.equals(EntityType.TENANT_PROFILE)
|| entityType.equals(EntityType.DEVICE_PROFILE)
|| entityType.equals(EntityType.API_USAGE_STATE)
|| (entityType.equals(EntityType.DEVICE) && msg.getEvent() == ComponentLifecycleEvent.UPDATED)
|| entityType.equals(EntityType.ENTITY_VIEW)
|| entityType.equals(EntityType.EDGE)) {
TbQueueProducer<TbProtoQueueMsg<ToCoreNotificationMsg>> toCoreNfProducer = producerProvider.getTbCoreNotificationsMsgProducer();
Set<String> tbCoreServices = partitionService.getAllServiceIds(ServiceType.TB_CORE);

18
application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java

@ -28,6 +28,7 @@ import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.actors.ActorSystemContext;
import org.thingsboard.server.common.data.alarm.Alarm;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.rpc.RpcError;
import org.thingsboard.server.common.msg.MsgType;
@ -236,6 +237,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
} else if (toCoreMsg.hasEdgeNotificationMsg()) {
log.trace("[{}] Forwarding message to edge service {}", id, toCoreMsg.getEdgeNotificationMsg());
forwardToEdgeNotificationService(toCoreMsg.getEdgeNotificationMsg(), callback);
} else if (toCoreMsg.hasDeviceActivityMsg()) {
log.trace("[{}] Forwarding message to device state service {}", id, toCoreMsg.getDeviceActivityMsg());
forwardToStateService(toCoreMsg.getDeviceActivityMsg(), callback);
} else if (!toCoreMsg.getToDeviceActorNotificationMsg().isEmpty()) {
Optional<TbActorMsg> actorMsg = encodingService.decode(toCoreMsg.getToDeviceActorNotificationMsg().toByteArray());
if (actorMsg.isPresent()) {
@ -520,6 +524,20 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService<ToCore
stateService.onQueueMsg(deviceStateServiceMsg, callback);
}
private void forwardToStateService(TransportProtos.DeviceActivityProto deviceActivityMsg, TbCallback callback) {
if (statsEnabled) {
stats.log(deviceActivityMsg);
}
TenantId tenantId = TenantId.fromUUID(new UUID(deviceActivityMsg.getTenantIdMSB(), deviceActivityMsg.getTenantIdLSB()));
DeviceId deviceId = new DeviceId(new UUID(deviceActivityMsg.getDeviceIdMSB(), deviceActivityMsg.getDeviceIdLSB()));
try {
stateService.onDeviceActivity(tenantId, deviceId, deviceActivityMsg.getLastActivityTime());
callback.onSuccess();
} catch (Exception e) {
callback.onFailure(new RuntimeException("Failed update device activity for device [" + deviceId.getId() + "]!", e));
}
}
private void forwardToEdgeNotificationService(EdgeNotificationMsgProto edgeNotificationMsg, TbCallback callback) {
if (statsEnabled) {
stats.log(edgeNotificationMsg);

8
application/src/main/java/org/thingsboard/server/service/queue/TbCoreConsumerStats.java

@ -38,6 +38,7 @@ public class TbCoreConsumerStats {
public static final String SUBSCRIPTION_MSGS = "subMsgs";
public static final String TO_CORE_NOTIFICATIONS = "coreNfs";
public static final String EDGE_NOTIFICATIONS = "edgeNfs";
public static final String DEVICE_ACTIVITIES = "deviceActivity";
private final StatsCounter totalCounter;
private final StatsCounter sessionEventCounter;
@ -52,6 +53,7 @@ public class TbCoreConsumerStats {
private final StatsCounter subscriptionMsgCounter;
private final StatsCounter toCoreNotificationsCounter;
private final StatsCounter edgeNotificationsCounter;
private final StatsCounter deviceActivitiesCounter;
private final List<StatsCounter> counters = new ArrayList<>();
@ -70,6 +72,7 @@ public class TbCoreConsumerStats {
this.subscriptionMsgCounter = register(statsFactory.createStatsCounter(statsKey, SUBSCRIPTION_MSGS));
this.toCoreNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, TO_CORE_NOTIFICATIONS));
this.edgeNotificationsCounter = register(statsFactory.createStatsCounter(statsKey, EDGE_NOTIFICATIONS));
this.deviceActivitiesCounter = register(statsFactory.createStatsCounter(statsKey, DEVICE_ACTIVITIES));
}
private StatsCounter register(StatsCounter counter){
@ -112,6 +115,11 @@ public class TbCoreConsumerStats {
edgeNotificationsCounter.increment();
}
public void log(TransportProtos.DeviceActivityProto msg) {
totalCounter.increment();
deviceActivitiesCounter.increment();
}
public void log(TransportProtos.SubscriptionMgrMsgProto msg) {
totalCounter.increment();
subscriptionMsgCounter.increment();

2
application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java

@ -180,6 +180,8 @@ public abstract class AbstractConsumerService<N extends com.google.protobuf.Gene
deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceProfileId(componentLifecycleMsg.getEntityId().getId()));
} else if (EntityType.DEVICE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
deviceProfileCache.evict(componentLifecycleMsg.getTenantId(), new DeviceId(componentLifecycleMsg.getEntityId().getId()));
} else if (EntityType.ENTITY_VIEW.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
actorContext.getTbEntityViewService().onComponentLifecycleMsg(componentLifecycleMsg);
} else if (EntityType.API_USAGE_STATE.equals(componentLifecycleMsg.getEntityId().getEntityType())) {
apiUsageStateService.onApiUsageStateUpdate(componentLifecycleMsg.getTenantId());
} else if (EntityType.CUSTOMER.equals(componentLifecycleMsg.getEntityId().getEntityType())) {

9
application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java

@ -34,6 +34,7 @@ import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.apiusage.RateLimitService;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService;
@ -57,9 +58,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
private final Map<EntityType, EntityExportService<?, ?, ?>> exportServices = new HashMap<>();
private final Map<EntityType, EntityImportService<?, ?, ?>> importServices = new HashMap<>();
private final EntityActionService entityActionService;
private final RelationService relationService;
private final RateLimitService rateLimitService;
private final TbNotificationEntityService entityNotificationService;
protected static final List<EntityType> SUPPORTED_ENTITY_TYPES = List.of(
EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN,
@ -109,10 +110,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS
relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations()));
for (EntityRelation relation : ctx.getRelations()) {
entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null,
ActionType.RELATION_ADD_OR_UPDATE, null, relation);
entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null,
ActionType.RELATION_ADD_OR_UPDATE, null, relation);
entityNotificationService.notifyCreateOrUpdateOrDeleteRelation(ctx.getTenantId(), null,
relation, ctx.getUser(), ActionType.RELATION_ADD_OR_UPDATE, null, relation);
}
}

2
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java

@ -41,8 +41,6 @@ public abstract class BaseEntityExportService<I extends EntityId, E extends Expo
return (D) new EntityExportData<E>();
}
;
public abstract Set<EntityType> getSupportedEntityTypes();
protected void replaceUuidsRecursively(EntitiesExportCtx<?> ctx, JsonNode node, Set<String> skipFieldsSet) {

8
application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java

@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.dao.relation.RelationDao;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.sync.ie.exporting.EntityExportService;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
@ -55,7 +55,7 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
@Lazy
protected ExportableEntitiesService exportableEntitiesService;
@Autowired
private RelationService relationService;
private RelationDao relationDao;
@Autowired
private AttributesService attributesService;
@ -99,10 +99,10 @@ public class DefaultEntityExportService<I extends EntityId, E extends Exportable
private List<EntityRelation> exportRelations(EntitiesExportCtx<?> ctx, E entity) throws ThingsboardException {
List<EntityRelation> relations = new ArrayList<>();
List<EntityRelation> inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
List<EntityRelation> inboundRelations = relationDao.findAllByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
relations.addAll(inboundRelations);
List<EntityRelation> outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
List<EntityRelation> outboundRelations = relationDao.findAllByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON);
relations.addAll(outboundRelations);
return relations;
}

11
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java

@ -19,14 +19,11 @@ import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.asset.Asset;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.AssetId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.asset.AssetService;
import org.thingsboard.server.queue.util.TbCoreComponent;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
@Service
@ -52,14 +49,6 @@ public class AssetImportService extends BaseEntityImportService<AssetId, Asset,
return assetService.saveAsset(asset);
}
@Override
protected void onEntitySaved(SecurityUser user, Asset savedAsset, Asset oldAsset) throws ThingsboardException {
super.onEntitySaved(user, savedAsset, oldAsset);
if (oldAsset != null) {
entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedAsset.getId(), EdgeEventActionType.UPDATED);
}
}
@Override
protected Asset deepCopy(Asset asset) {
return new Asset(asset);

24
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/BaseEntityImportService.java

@ -26,10 +26,8 @@ import org.springframework.context.annotation.Lazy;
import org.springframework.transaction.annotation.Transactional;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
import org.thingsboard.server.common.data.HasCustomerId;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
@ -49,13 +47,13 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup;
import org.thingsboard.server.common.data.sync.ie.AttributeExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.dao.relation.RelationDao;
import org.thingsboard.server.dao.relation.RelationService;
import org.thingsboard.server.service.action.EntityActionService;
import org.thingsboard.server.service.entitiy.TbNotificationEntityService;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService;
import org.thingsboard.server.service.sync.ie.importing.EntityImportService;
import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx;
import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx;
import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService;
@ -79,6 +77,8 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
@Autowired
private RelationService relationService;
@Autowired
private RelationDao relationDao;
@Autowired
private TelemetrySubscriptionService tsSubService;
@Autowired
protected EntityActionService entityActionService;
@ -203,19 +203,18 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
if (importResult.getOldEntity() != null) {
List<EntityRelation> existingRelations = new ArrayList<>();
existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationDao.findAllByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON));
existingRelations.addAll(relationDao.findAllByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON));
// dao is used here instead of service to avoid getting cached values, because relationService.deleteRelation will evict value from cache only after transaction is committed
for (EntityRelation existingRelation : existingRelations) {
EntityRelation relation = relationsMap.get(existingRelation);
if (relation == null) {
importResult.setUpdatedRelatedEntities(true);
relationService.deleteRelation(tenantId, existingRelation);
relationService.deleteRelation(ctx.getTenantId(), existingRelation.getFrom(), existingRelation.getTo(), existingRelation.getType(), existingRelation.getTypeGroup());
importResult.addSendEventsCallback(() -> {
entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null,
ActionType.RELATION_DELETED, null, existingRelation);
entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null,
ActionType.RELATION_DELETED, null, existingRelation);
entityNotificationService.notifyCreateOrUpdateOrDeleteRelation(tenantId, null,
existingRelation, ctx.getUser(), ActionType.RELATION_DELETED, null, existingRelation);
});
} else if (Objects.equal(relation.getAdditionalInfo(), existingRelation.getAdditionalInfo())) {
relationsMap.remove(relation);
@ -269,9 +268,8 @@ public abstract class BaseEntityImportService<I extends EntityId, E extends Expo
}
protected void onEntitySaved(SecurityUser user, E savedEntity, E oldEntity) throws ThingsboardException {
entityActionService.logEntityAction(user, savedEntity.getId(), savedEntity,
savedEntity instanceof HasCustomerId ? ((HasCustomerId) savedEntity).getCustomerId() : user.getCustomerId(),
oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, null);
entityNotificationService.notifyCreateOrUpdateEntity(user.getTenantId(), savedEntity.getId(), savedEntity,
null, oldEntity == null ? ActionType.ADDED : ActionType.UPDATED, user);
}

8
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/CustomerImportService.java

@ -68,14 +68,6 @@ public class CustomerImportService extends BaseEntityImportService<CustomerId, C
return new Customer(customer);
}
@Override
protected void onEntitySaved(SecurityUser user, Customer savedCustomer, Customer oldCustomer) throws ThingsboardException {
super.onEntitySaved(user, savedCustomer, oldCustomer);
if (oldCustomer != null) {
entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedCustomer.getId(), EdgeEventActionType.UPDATED);
}
}
@Override
public EntityType getEntityType() {
return EntityType.CUSTOMER;

7
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DashboardImportService.java

@ -118,11 +118,8 @@ public class DashboardImportService extends BaseEntityImportService<DashboardId,
}
@Override
protected void onEntitySaved(SecurityUser user, Dashboard savedDashboard, Dashboard oldDashboard) throws ThingsboardException {
super.onEntitySaved(user, savedDashboard, oldDashboard);
if (oldDashboard != null) {
entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedDashboard.getId(), EdgeEventActionType.UPDATED);
}
protected boolean compare(EntitiesImportCtx ctx, EntityExportData<Dashboard> exportData, Dashboard prepared, Dashboard existing) {
return super.compare(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration());
}
@Override

5
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java

@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.Device;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.data.id.TenantId;
@ -93,8 +94,8 @@ public class DeviceImportService extends BaseEntityImportService<DeviceId, Devic
@Override
protected void onEntitySaved(SecurityUser user, Device savedDevice, Device oldDevice) throws ThingsboardException {
super.onEntitySaved(user, savedDevice, oldDevice);
clusterService.onDeviceUpdated(savedDevice, oldDevice);
entityNotificationService.notifyCreateOrUpdateDevice(user.getTenantId(), savedDevice.getId(), savedDevice.getCustomerId(),
savedDevice, oldDevice, oldDevice == null ? ActionType.ADDED : ActionType.UPDATED, user);
}
@Override

7
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceProfileImportService.java

@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.DeviceProfile;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.DeviceProfileId;
@ -52,6 +53,7 @@ public class DeviceProfileImportService extends BaseEntityImportService<DevicePr
deviceProfile.setDefaultDashboardId(idProvider.getInternalId(deviceProfile.getDefaultDashboardId()));
deviceProfile.setFirmwareId(getOldEntityField(old, DeviceProfile::getFirmwareId));
deviceProfile.setSoftwareId(getOldEntityField(old, DeviceProfile::getSoftwareId));
deviceProfile.setDefaultQueueId(getOldEntityField(old, DeviceProfile::getDefaultQueueId));
return deviceProfile;
}
@ -62,15 +64,14 @@ public class DeviceProfileImportService extends BaseEntityImportService<DevicePr
@Override
protected void onEntitySaved(SecurityUser user, DeviceProfile savedDeviceProfile, DeviceProfile oldDeviceProfile) throws ThingsboardException {
super.onEntitySaved(user, savedDeviceProfile, oldDeviceProfile);
clusterService.onDeviceProfileChange(savedDeviceProfile, null);
clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedDeviceProfile.getId(),
oldDeviceProfile == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedDeviceProfile.getId(),
oldDeviceProfile == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED);
otaPackageStateService.update(savedDeviceProfile,
oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getFirmwareId(), savedDeviceProfile.getFirmwareId()),
oldDeviceProfile != null && !Objects.equals(oldDeviceProfile.getSoftwareId(), savedDeviceProfile.getSoftwareId()));
entityNotificationService.notifyCreateOrUpdateOrDelete(savedDeviceProfile.getTenantId(), null,
savedDeviceProfile.getId(), savedDeviceProfile, user, oldDeviceProfile == null ? ActionType.ADDED : ActionType.UPDATED, true, null);
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/EntityViewImportService.java

@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.queue.util.TbCoreComponent;
@ -64,9 +65,8 @@ public class EntityViewImportService extends BaseEntityImportService<EntityViewI
protected void onEntitySaved(SecurityUser user, EntityView savedEntityView, EntityView oldEntityView) throws ThingsboardException {
tbEntityViewService.updateEntityViewAttributes(user, savedEntityView, oldEntityView);
super.onEntitySaved(user, savedEntityView, oldEntityView);
if (oldEntityView != null) {
entityActionService.sendEntityNotificationMsgToEdge(user.getTenantId(), savedEntityView.getId(), EdgeEventActionType.UPDATED);
}
clusterService.broadcastEntityStateChangeEvent(savedEntityView.getTenantId(), savedEntityView.getId(),
oldEntityView == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);
}
@Override

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/RuleChainImportService.java

@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.audit.ActionType;
import org.thingsboard.server.common.data.edge.EdgeEventActionType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.RuleChainId;
@ -119,14 +120,15 @@ public class RuleChainImportService extends BaseEntityImportService<RuleChainId,
RuleChainMetaData newMD = exportData.getMetaData();
RuleChainMetaData existingMD = ruleChainService.loadRuleChainMetaData(ctx.getTenantId(), prepared.getId());
existingMD.setRuleChainId(null);
different = newMD.equals(existingMD);
different = !newMD.equals(existingMD);
}
return different;
}
@Override
protected void onEntitySaved(SecurityUser user, RuleChain savedRuleChain, RuleChain oldRuleChain) throws ThingsboardException {
super.onEntitySaved(user, savedRuleChain, oldRuleChain);
entityActionService.logEntityAction(user, savedRuleChain.getId(), savedRuleChain, null,
oldRuleChain == null ? ActionType.ADDED : ActionType.UPDATED, null);
if (savedRuleChain.getType() == RuleChainType.CORE) {
clusterService.broadcastEntityStateChangeEvent(user.getTenantId(), savedRuleChain.getId(),
oldRuleChain == null ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED);

6
application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/WidgetsBundleImportService.java

@ -86,9 +86,13 @@ public class WidgetsBundleImportService extends BaseEntityImportService<WidgetsB
return savedWidgetsBundle;
}
@Override
protected boolean compare(EntitiesImportCtx ctx, WidgetsBundleExportData exportData, WidgetsBundle prepared, WidgetsBundle existing) {
return true;
}
@Override
protected void onEntitySaved(SecurityUser user, WidgetsBundle savedWidgetsBundle, WidgetsBundle oldWidgetsBundle) throws ThingsboardException {
super.onEntitySaved(user, savedWidgetsBundle, oldWidgetsBundle);
entityNotificationService.notifySendMsgToEdgeService(user.getTenantId(), savedWidgetsBundle.getId(),
oldWidgetsBundle == null ? EdgeEventActionType.ADDED : EdgeEventActionType.UPDATED);
}

26
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java

@ -25,12 +25,10 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.transaction.support.TransactionTemplate;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.common.util.DonAsynchron;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.common.util.TbStopWatch;
import org.thingsboard.common.util.ThingsBoardExecutors;
import org.thingsboard.server.cache.CaffeineTbTransactionalCache;
import org.thingsboard.server.cache.TbTransactionalCache;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.ExportableEntity;
@ -49,6 +47,7 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.ie.EntityExportSettings;
import org.thingsboard.server.common.data.sync.ie.EntityImportResult;
import org.thingsboard.server.common.data.sync.ie.EntityImportSettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.EntityLoadError;
@ -175,9 +174,11 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
private <T> T getStatus(SecurityUser user, UUID requestId, Function<VersionControlTaskCacheEntry, T> getter) throws ThingsboardException {
var cacheEntry = taskCache.get(requestId);
if (cacheEntry == null || cacheEntry.get() == null) {
log.debug("[{}] No cache record: {}", requestId, cacheEntry);
throw new ThingsboardException(ThingsboardErrorCode.ITEM_NOT_FOUND);
} else {
var entry = cacheEntry.get();
log.debug("[{}] Cache get: {}", requestId, entry);
var result = getter.apply(entry);
if (result == null) {
throw new ThingsboardException(ThingsboardErrorCode.BAD_REQUEST_PARAMS);
@ -358,10 +359,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
return EntityImportSettings.builder()
.updateRelations(config.isLoadRelations())
.saveAttributes(config.isLoadAttributes())
.saveCredentials(config.isLoadCredentials())
.findExistingByName(config.isFindExistingEntityByName())
.build();
}
@SneakyThrows
@SuppressWarnings({"rawtypes", "unchecked"})
private void importEntities(EntitiesImportCtx ctx, EntityType entityType) {
int limit = 100;
@ -373,9 +376,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e);
}
log.debug("[{}] Loading {} entities pack ({})", ctx.getTenantId(), entityType, entityDataList.size());
for (EntityExportData entityData : entityDataList) {
EntityExportData reimportBackup = JacksonUtil.clone(entityData);
log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType);
EntityImportResult<?> importResult;
try {
importResult = exportImportService.importEntity(ctx, entityData);
@ -391,6 +394,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>())
.add(importResult.getSavedEntity().getId());
}
log.debug("Imported {} pack for tenant {}", entityType, ctx.getTenantId());
offset += limit;
} while (entityDataList.size() == limit);
}
@ -456,18 +460,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
EntityId externalId = ((ExportableEntity<EntityId>) entity).getExternalId();
if (externalId == null) externalId = entityId;
return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId),
return transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId),
otherVersion -> {
SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null, EntityExportSettings.builder()
.exportRelations(otherVersion.hasRelations())
.exportAttributes(otherVersion.hasAttributes())
.exportCredentials(otherVersion.hasCredentials())
.build());
EntityExportData<?> currentVersion = exportImportService.exportEntity(ctx, entityId);
return transform(gitServiceQueue.getContentsDiff(user.getTenantId(),
JacksonUtil.toPrettyString(currentVersion.sort()),
JacksonUtil.toPrettyString(otherVersion.sort())),
rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor());
EntityExportData<?> currentVersion;
try {
currentVersion = exportImportService.exportEntity(ctx, entityId);
} catch (ThingsboardException e) {
throw new RuntimeException(e);
}
return new EntityDataDiff(currentVersion.sort(), otherVersion.sort());
}, MoreExecutors.directExecutor());
}
@ -479,7 +485,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont
@Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception {
public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception {
return gitServiceQueue.listBranches(tenantId);
}

143
application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java

@ -15,7 +15,10 @@
*/
package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.common.util.concurrent.SettableFuture;
import com.google.protobuf.ByteString;
import lombok.SneakyThrows;
@ -23,6 +26,7 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.CollectionsUtil;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.cluster.TbClusterService;
import org.thingsboard.server.common.data.EntityType;
@ -35,6 +39,7 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
@ -70,12 +75,16 @@ import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest;
import org.thingsboard.server.service.sync.vc.data.VoidGitRequest;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -92,9 +101,12 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
private final SchedulerComponent scheduler;
private final Map<UUID, PendingGitRequest<?>> pendingRequestMap = new HashMap<>();
private final Map<UUID, Map<String, String[]>> chunkedMsgs = new ConcurrentHashMap<>();
@Value("${queue.vc.request-timeout:60000}")
private int requestTimeout;
@Value("${queue.vc.msg-chunk-size:500000}")
private int msgChunkSize;
public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService,
DataDecodingEncodingService encodingService,
@ -118,20 +130,35 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
return future;
}
@SuppressWarnings("UnstableApiUsage")
@Override
public ListenableFuture<Void> addToCommit(CommitGitRequest commit, EntityExportData<ExportableEntity<EntityId>> entityData) {
SettableFuture<Void> future = SettableFuture.create();
String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId());
String entityDataJson = JacksonUtil.toPrettyString(entityData.sort());
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setAddMsg(
TransportProtos.AddMsg.newBuilder()
.setRelativePath(path).setEntityDataJson(entityDataJson).build()
).build()
).build(), wrap(future, null));
return future;
Iterable<String> entityDataChunks = StringUtils.split(entityDataJson, msgChunkSize);
String chunkedMsgId = UUID.randomUUID().toString();
int chunksCount = Iterables.size(entityDataChunks);
AtomicInteger chunkIndex = new AtomicInteger();
List<ListenableFuture<Void>> futures = new ArrayList<>();
entityDataChunks.forEach(chunk -> {
SettableFuture<Void> chunkFuture = SettableFuture.create();
log.trace("[{}] sending chunk {} for 'addToCommit'", chunkedMsgId, chunkIndex.get());
registerAndSend(commit, builder -> builder.setCommitRequest(
buildCommitRequest(commit).setAddMsg(
TransportProtos.AddMsg.newBuilder()
.setRelativePath(path).setEntityDataJsonChunk(chunk)
.setChunkedMsgId(chunkedMsgId).setChunkIndex(chunkIndex.getAndIncrement())
.setChunksCount(chunksCount).build()
).build()
).build(), wrap(chunkFuture, null));
futures.add(chunkFuture);
});
return Futures.transform(Futures.allAsList(futures), r -> {
log.trace("[{}] sent all chunks for 'addToCommit'", chunkedMsgId);
return null;
}, MoreExecutors.directExecutor());
}
@Override
@ -220,7 +247,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
@Override
public ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) {
return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder()
.setBranchName(branch)
.setVersionId(versionId)
.setEntityType(entityType.name())
.build());
@ -229,7 +255,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
@Override
public ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) {
return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder()
.setBranchName(branch)
.setVersionId(versionId)
.build());
}
@ -240,7 +265,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
}
@Override
public ListenableFuture<List<String>> listBranches(TenantId tenantId) {
public ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) {
ListBranchesGitRequest request = new ListBranchesGitRequest(tenantId);
return sendRequest(request, builder -> builder.setListBranchesRequest(TransportProtos.ListBranchesRequestMsg.newBuilder().build()));
}
@ -256,18 +281,11 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
.build()));
}
@Override
public ListenableFuture<String> getContentsDiff(TenantId tenantId, String content1, String content2) {
ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2);
return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder()
.setContent1(content1)
.setContent2(content2)));
}
@Override
@SuppressWarnings("rawtypes")
public ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId) {
EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId);
chunkedMsgs.put(request.getRequestId(), new HashMap<>());
registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder()
.setVersionId(versionId)
.setEntityType(entityId.getEntityType().name())
@ -289,9 +307,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
var requestBody = enrichFunction.apply(newRequestProto(request, settings));
log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody);
clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback);
request.setTimeoutTask(scheduler.schedule(() -> {
processTimeout(request.getRequestId());
}, requestTimeout, TimeUnit.MILLISECONDS));
if (request.getTimeoutTask() == null) {
request.setTimeoutTask(scheduler.schedule(() -> processTimeout(request.getRequestId()), requestTimeout, TimeUnit.MILLISECONDS));
}
} else {
throw new RuntimeException("Future is already done!");
}
@ -309,7 +327,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
@SuppressWarnings("rawtypes")
public ListenableFuture<List<EntityExportData>> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) {
EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType);
chunkedMsgs.put(request.getRequestId(), new HashMap<>());
registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder()
.setVersionId(versionId)
.setEntityType(entityType.name())
@ -355,15 +373,15 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
@Override
public void processResponse(VersionControlResponseMsg vcResponseMsg) {
UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB());
PendingGitRequest<?> request = pendingRequestMap.remove(requestId);
PendingGitRequest<?> request = pendingRequestMap.get(requestId);
if (request == null) {
log.debug("[{}] received stale response: {}", requestId, vcResponseMsg);
return;
} else {
log.debug("[{}] processing response: {}", requestId, vcResponseMsg);
request.getTimeoutTask().cancel(true);
}
var future = request.getFuture();
boolean completed = true;
if (!StringUtils.isEmpty(vcResponseMsg.getError())) {
future.setException(new RuntimeException(vcResponseMsg.getError()));
} else {
@ -382,7 +400,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
((CommitGitRequest) request).getFuture().set(commitResult);
} else if (vcResponseMsg.hasListBranchesResponse()) {
var listBranchesResponse = vcResponseMsg.getListBranchesResponse();
((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList());
((ListBranchesGitRequest) request).getFuture().set(listBranchesResponse.getBranchesList().stream().map(this::getBranchInfo).collect(Collectors.toList()));
} else if (vcResponseMsg.hasListEntitiesResponse()) {
var listEntitiesResponse = vcResponseMsg.getListEntitiesResponse();
((ListEntitiesGitRequest) request).getFuture().set(
@ -391,12 +409,28 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
var listVersionsResponse = vcResponseMsg.getListVersionsResponse();
((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse));
} else if (vcResponseMsg.hasEntityContentResponse()) {
var data = vcResponseMsg.getEntityContentResponse().getData();
((EntityContentGitRequest) request).getFuture().set(toData(data));
TransportProtos.EntityContentResponseMsg responseMsg = vcResponseMsg.getEntityContentResponse();
log.trace("[{}] received chunk {} for 'getEntity'", responseMsg.getChunkedMsgId(), responseMsg.getChunkIndex());
var joined = joinChunks(requestId, responseMsg, 1);
if (joined.isPresent()) {
log.trace("[{}] collected all chunks for 'getEntity'", responseMsg.getChunkedMsgId());
((EntityContentGitRequest) request).getFuture().set(joined.get().get(0));
} else {
completed = false;
}
} else if (vcResponseMsg.hasEntitiesContentResponse()) {
var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList();
((EntitiesContentGitRequest) request).getFuture()
.set(dataList.stream().map(this::toData).collect(Collectors.toList()));
TransportProtos.EntitiesContentResponseMsg responseMsg = vcResponseMsg.getEntitiesContentResponse();
TransportProtos.EntityContentResponseMsg item = responseMsg.getItem();
if (responseMsg.getItemsCount() > 0) {
var joined = joinChunks(requestId, item, responseMsg.getItemsCount());
if (joined.isPresent()) {
((EntitiesContentGitRequest) request).getFuture().set(joined.get());
} else {
completed = false;
}
} else {
((EntitiesContentGitRequest) request).getFuture().set(Collections.emptyList());
}
} else if (vcResponseMsg.hasVersionsDiffResponse()) {
TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse();
List<EntityVersionsDiff> entityVersionsDiffList = diffResponse.getDiffList().stream()
@ -411,21 +445,50 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
.build())
.collect(Collectors.toList());
((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList);
} else if (vcResponseMsg.hasContentsDiffResponse()) {
String diff = vcResponseMsg.getContentsDiffResponse().getDiff();
((ContentsDiffGitRequest) request).getFuture().set(diff);
}
}
if (completed) {
removePendingRequest(requestId);
}
}
@SuppressWarnings("rawtypes")
private Optional<List<EntityExportData>> joinChunks(UUID requestId, TransportProtos.EntityContentResponseMsg responseMsg, int expectedMsgCount) {
var chunksMap = chunkedMsgs.get(requestId);
if (chunksMap == null) {
return Optional.empty();
}
String[] msgChunks = chunksMap.computeIfAbsent(responseMsg.getChunkedMsgId(), id -> new String[responseMsg.getChunksCount()]);
msgChunks[responseMsg.getChunkIndex()] = responseMsg.getData();
if (chunksMap.size() == expectedMsgCount && chunksMap.values().stream()
.allMatch(chunks -> CollectionsUtil.countNonNull(chunks) == chunks.length)) {
return Optional.of(chunksMap.values().stream()
.map(chunks -> String.join("", chunks))
.map(this::toData)
.collect(Collectors.toList()));
} else {
return Optional.empty();
}
}
private void processTimeout(UUID requestId) {
PendingGitRequest<?> pendingRequest = pendingRequestMap.remove(requestId);
PendingGitRequest<?> pendingRequest = removePendingRequest(requestId);
if (pendingRequest != null) {
log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout);
pendingRequest.getFuture().setException(new TimeoutException("Request timed out"));
}
}
private PendingGitRequest<?> removePendingRequest(UUID requestId) {
PendingGitRequest<?> pendingRequest = pendingRequestMap.remove(requestId);
if (pendingRequest != null && pendingRequest.getTimeoutTask() != null) {
pendingRequest.getTimeoutTask().cancel(true);
pendingRequest.setTimeoutTask(null);
}
chunkedMsgs.remove(requestId);
return pendingRequest;
}
private PageData<EntityVersion> toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) {
var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList());
return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext());
@ -439,12 +502,17 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
return new VersionedEntityInfo(EntityIdFactory.getByTypeAndUuid(proto.getEntityType(), new UUID(proto.getEntityIdMSB(), proto.getEntityIdLSB())));
}
private BranchInfo getBranchInfo(TransportProtos.BranchInfoProto proto) {
return new BranchInfo(proto.getName(), proto.getIsDefault());
}
@SuppressWarnings("rawtypes")
@SneakyThrows
private EntityExportData toData(String data) {
return JacksonUtil.fromString(data, EntityExportData.class);
}
//The future will be completed when the corresponding result arrives from kafka
private static <T> TbQueueCallback wrap(SettableFuture<T> future) {
return new TbQueueCallback() {
@Override
@ -458,7 +526,8 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu
};
}
private static <T> TbQueueCallback wrap(SettableFuture<T> future, T value) {
//The future will be completed when the request is successfully sent to kafka
private <T> TbQueueCallback wrap(SettableFuture<T> future, T value) {
return new TbQueueCallback() {
@Override
public void onSuccess(TbQueueMsgMetadata metadata) {

11
application/src/main/java/org/thingsboard/server/service/sync/vc/EntitiesVersionControlService.java

@ -16,22 +16,21 @@
package org.thingsboard.server.service.sync.vc;
import com.google.common.util.concurrent.ListenableFuture;
import org.springframework.web.context.request.async.DeferredResult;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.exception.ThingsboardException;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityDataDiff;
import org.thingsboard.server.common.data.sync.vc.EntityDataInfo;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.EntityTypeLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionLoadResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.security.model.SecurityUser;
import org.thingsboard.server.common.data.sync.vc.request.load.VersionLoadRequest;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
@ -60,7 +59,7 @@ public interface EntitiesVersionControlService {
ListenableFuture<EntityDataDiff> compareEntityDataToVersion(SecurityUser user, String branch, EntityId entityId, String versionId) throws Exception;
ListenableFuture<List<String>> listBranches(TenantId tenantId) throws Exception;
ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId) throws Exception;
RepositorySettings getVersionControlSettings(TenantId tenantId);

9
application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java

@ -24,14 +24,15 @@ import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.ie.EntityExportData;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.common.data.sync.vc.request.create.VersionCreateRequest;
import org.thingsboard.server.gen.transport.TransportProtos.VersionControlResponseMsg;
import org.thingsboard.server.service.sync.vc.data.CommitGitRequest;
import org.thingsboard.server.common.data.sync.vc.EntityVersionsDiff;
import java.util.List;
@ -55,7 +56,7 @@ public interface GitVersionControlQueueService {
ListenableFuture<List<VersionedEntityInfo>> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId);
ListenableFuture<List<String>> listBranches(TenantId tenantId);
ListenableFuture<List<BranchInfo>> listBranches(TenantId tenantId);
ListenableFuture<EntityExportData> getEntity(TenantId tenantId, String versionId, EntityId entityId);
@ -63,8 +64,6 @@ public interface GitVersionControlQueueService {
ListenableFuture<List<EntityVersionsDiff>> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2);
ListenableFuture<String> getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2);
ListenableFuture<Void> initRepository(TenantId tenantId, RepositorySettings settings);
ListenableFuture<Void> testRepository(TenantId tenantId, RepositorySettings settings);

3
application/src/main/java/org/thingsboard/server/service/sync/vc/data/ListBranchesGitRequest.java

@ -16,10 +16,11 @@
package org.thingsboard.server.service.sync.vc.data;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import java.util.List;
public class ListBranchesGitRequest extends PendingGitRequest<List<String>> {
public class ListBranchesGitRequest extends PendingGitRequest<List<BranchInfo>> {
public ListBranchesGitRequest(TenantId tenantId) {
super(tenantId);

13
application/src/main/java/org/thingsboard/server/service/telemetry/DefaultTelemetrySubscriptionService.java

@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.MoreExecutors;
import lombok.extern.slf4j.Slf4j;
import org.jetbrains.annotations.NotNull;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.cluster.TbClusterService;
@ -43,12 +44,12 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TbCallback;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.dao.attributes.AttributesService;
import org.thingsboard.server.dao.entityview.EntityViewService;
import org.thingsboard.server.dao.timeseries.TimeseriesService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.queue.discovery.PartitionService;
import org.thingsboard.server.queue.usagestats.TbApiUsageClient;
import org.thingsboard.server.service.apiusage.TbApiUsageStateService;
import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService;
import org.thingsboard.server.service.subscription.TbSubscriptionUtils;
import javax.annotation.Nullable;
@ -75,7 +76,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
private final AttributesService attrService;
private final TimeseriesService tsService;
private final EntityViewService entityViewService;
private final TbEntityViewService tbEntityViewService;
private final TbApiUsageClient apiUsageClient;
private final TbApiUsageStateService apiUsageStateService;
@ -83,7 +84,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
public DefaultTelemetrySubscriptionService(AttributesService attrService,
TimeseriesService tsService,
EntityViewService entityViewService,
@Lazy TbEntityViewService tbEntityViewService,
TbClusterService clusterService,
PartitionService partitionService,
TbApiUsageClient apiUsageClient,
@ -91,7 +92,7 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
super(clusterService, partitionService);
this.attrService = attrService;
this.tsService = tsService;
this.entityViewService = entityViewService;
this.tbEntityViewService = tbEntityViewService;
this.apiUsageClient = apiUsageClient;
this.apiUsageStateService = apiUsageStateService;
}
@ -182,11 +183,11 @@ public class DefaultTelemetrySubscriptionService extends AbstractSubscriptionSer
addMainCallback(saveFuture, callback);
addWsCallback(saveFuture, success -> onTimeSeriesUpdate(tenantId, entityId, ts));
if (EntityType.DEVICE.equals(entityId.getEntityType()) || EntityType.ASSET.equals(entityId.getEntityType())) {
Futures.addCallback(this.entityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId),
Futures.addCallback(this.tbEntityViewService.findEntityViewsByTenantIdAndEntityIdAsync(tenantId, entityId),
new FutureCallback<List<EntityView>>() {
@Override
public void onSuccess(@Nullable List<EntityView> result) {
if (result != null) {
if (result != null && !result.isEmpty()) {
Map<String, List<TsKvEntry>> tsMap = new HashMap<>();
for (TsKvEntry entry : ts) {
tsMap.computeIfAbsent(entry.getKey(), s -> new ArrayList<>()).add(entry);

39
application/src/main/resources/logback.xml

@ -25,26 +25,31 @@
</encoder>
</appender>
<logger name="org.thingsboard.server" level="INFO" />
<!-- <logger name="org.springframework.jdbc.core" level="TRACE" />-->
<!-- <logger name="org.hibernate.SQL" level="DEBUG" />-->
<!-- <logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.queue.memory.InMemoryStorage" level="DEBUG" />-->
<!-- <logger name="org.thingsboard.server.service.ttl.AbstractCleanUpService" level="DEBUG" />-->
<!-- <logger name="org.thingsboard.server.service.subscription" level="TRACE"/>-->
<!-- <logger name="org.thingsboard.server.service.telemetry" level="TRACE"/>-->
<!-- <logger name="org.eclipse.californium.scandium.DTLSConnector" level="TRACE" />-->
<!-- <logger name="org.eclipse.californium.scandium.dtls.Handshaker" level="DEBUG" />-->
<logger name="org.thingsboard.server" level="INFO"/>
<logger name="org.apache.kafka.common.utils.AppInfoParser" level="WARN"/>
<logger name="org.apache.kafka.clients.consumer.ConsumerConfig" level="WARN"/>
<!-- To enable the logging of scanned rule engine components-->
<!-- <logger name="org.thingsboard.server.service.component.AnnotationComponentDiscoveryService" level="DEBUG" />-->
<!-- Other useful logs -->
<!-- <logger name="org.springframework.jdbc.core" level="TRACE" />-->
<!-- <logger name="org.hibernate.SQL" level="DEBUG" />-->
<!-- <logger name="org.hibernate.type.descriptor.sql.BasicBinder" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.queue" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.service.transport" level="TRACE" />-->
<!-- <logger name="org.thingsboard.server.queue.memory.InMemoryStorage" level="DEBUG" />-->
<!-- <logger name="org.thingsboard.server.service.ttl.AbstractCleanUpService" level="DEBUG" />-->
<!-- <logger name="org.thingsboard.server.service.subscription" level="TRACE"/>-->
<!-- <logger name="org.thingsboard.server.service.telemetry" level="TRACE"/>-->
<!-- <logger name="org.eclipse.californium.scandium.DTLSConnector" level="TRACE" />-->
<!-- <logger name="org.eclipse.californium.scandium.dtls.Handshaker" level="DEBUG" />-->
<!-- Top Rule Nodes by max execution time -->
<!-- <logger name="org.thingsboard.server.service.queue.TbMsgPackProcessingContext" level="DEBUG" /> -->
<!-- <logger name="org.thingsboard.server.service.queue.TbMsgPackProcessingContext" level="DEBUG" /> -->
<!-- MQTT transport debug -->
<!-- <logger name="org.thingsboard.server.transport.mqtt.MqttTransportHandler" level="DEBUG" /> -->
<!-- MQTT transport debug -->
<!-- <logger name="org.thingsboard.server.transport.mqtt.MqttTransportHandler" level="DEBUG" /> -->
<logger name="com.microsoft.azure.servicebus.primitives.CoreMessageReceiver" level="OFF" />

4
application/src/main/resources/thingsboard.yml

@ -428,6 +428,9 @@ cache:
timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}"
maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}"
#Disable this because it is not required.
spring.data.redis.repositories.enabled: false
redis:
# standalone or cluster
connection:
@ -1027,6 +1030,7 @@ queue:
poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}"
pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}"
request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}"
msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:500000}"
js:
# JS Eval request topic
request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}"

2
application/src/test/java/org/thingsboard/server/controller/BaseEntityRelationControllerTest.java

@ -45,7 +45,7 @@ import java.util.List;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@Slf4j
public class BaseEntityRelationControllerTest extends AbstractControllerTest {
public abstract class BaseEntityRelationControllerTest extends AbstractControllerTest {
public static final String BASE_DEVICE_NAME = "Test dummy device";

29
application/src/test/java/org/thingsboard/server/edge/BaseEdgeTest.java

@ -1352,17 +1352,17 @@ abstract public class BaseEdgeTest extends AbstractControllerTest {
String timeseriesKey = "key";
String timeseriesValue = "25";
data.addProperty(timeseriesKey, timeseriesValue);
UplinkMsg.Builder uplinkMsgBuilder1 = UplinkMsg.newBuilder();
UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder();
EntityDataProto.Builder entityDataBuilder = EntityDataProto.newBuilder();
entityDataBuilder.setPostTelemetryMsg(JsonConverter.convertToTelemetryProto(data, System.currentTimeMillis()));
entityDataBuilder.setEntityType(device.getId().getEntityType().name());
entityDataBuilder.setEntityIdMSB(device.getUuidId().getMostSignificantBits());
entityDataBuilder.setEntityIdLSB(device.getUuidId().getLeastSignificantBits());
testAutoGeneratedCodeByProtobuf(entityDataBuilder);
uplinkMsgBuilder1.addEntityData(entityDataBuilder.build());
uplinkMsgBuilder.addEntityData(entityDataBuilder.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder1);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder1.build());
testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder);
edgeImitator.sendUplinkMsg(uplinkMsgBuilder.build());
JsonObject attributesData = new JsonObject();
String attributesKey = "test_attr";
@ -1394,9 +1394,24 @@ abstract public class BaseEdgeTest extends AbstractControllerTest {
String attributeValuesUrl = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/attributes/" + DataConstants.SERVER_SCOPE;
List<Map<String, String>> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() {});
Assert.assertEquals(2, attributes.size());
var result = attributes.stream().filter(kv -> kv.get("key").equals(attributesKey)).filter(kv -> kv.get("value").equals(attributesValue)).findFirst();
Assert.assertTrue(result.isPresent());
Assert.assertEquals(3, attributes.size());
Optional<Map<String, String>> activeAttributeOpt = getAttributeByKey("active", attributes);
Assert.assertTrue(activeAttributeOpt.isPresent());
Map<String, String> activeAttribute = activeAttributeOpt.get();
Assert.assertEquals("true", activeAttribute.get("value"));
Optional<Map<String, String>> customAttributeOpt = getAttributeByKey(attributesKey, attributes);
Assert.assertTrue(customAttributeOpt.isPresent());
Map<String, String> customAttribute = customAttributeOpt.get();
Assert.assertEquals(attributesValue, customAttribute.get("value"));
doDelete("/api/plugins/telemetry/DEVICE/" + device.getId().getId() + "/SERVER_SCOPE?keys=" + attributesKey, String.class);
}
private Optional<Map<String, String>> getAttributeByKey(String key, List<Map<String, String>> attributes) {
return attributes.stream().filter(kv -> kv.get("key").equals(key)).findFirst();
}
private Map<String, List<Map<String, String>>> loadDeviceTimeseries(Device device, String timeseriesKey) throws Exception {

20
application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java

@ -16,8 +16,6 @@
package org.thingsboard.server.transport.coap;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapClient;
import org.junit.After;
import org.springframework.test.context.TestPropertySource;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.Device;
@ -41,7 +39,6 @@ import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadCon
import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.transport.AbstractTransportIntegrationTest;
import static org.junit.Assert.assertEquals;
@ -54,12 +51,11 @@ import static org.junit.Assert.assertNotNull;
public abstract class AbstractCoapIntegrationTest extends AbstractTransportIntegrationTest {
protected final byte[] EMPTY_PAYLOAD = new byte[0];
protected CoapClient client;
protected CoapTestClient client;
protected void processAfterTest() throws Exception {
if (client != null) {
client.shutdown();
client.disconnect();
}
}
@ -156,16 +152,4 @@ public abstract class AbstractCoapIntegrationTest extends AbstractTransportInteg
device.setType(type);
return doPost("/api/device", device, Device.class);
}
protected CoapClient getCoapClient(FeatureType featureType) {
return new CoapClient(getFeatureTokenUrl(accessToken, featureType));
}
protected CoapClient getCoapClient(String featureTokenUrl) {
return new CoapClient(featureTokenUrl);
}
protected String getFeatureTokenUrl(String token, FeatureType featureType) {
return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase();
}
}

68
application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java

@ -0,0 +1,68 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.coap;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapHandler;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import java.util.concurrent.CountDownLatch;
@Slf4j
@Data
public class CoapTestCallback implements CoapHandler {
protected final CountDownLatch latch;
protected Integer observe;
protected byte[] payloadBytes;
protected CoAP.ResponseCode responseCode;
public CoapTestCallback() {
this.latch = new CountDownLatch(1);
}
public CoapTestCallback(int subscribeCount) {
this.latch = new CountDownLatch(subscribeCount);
}
public Integer getObserve() {
return observe;
}
public byte[] getPayloadBytes() {
return payloadBytes;
}
public CoAP.ResponseCode getResponseCode() {
return responseCode;
}
@Override
public void onLoad(CoapResponse response) {
observe = response.getOptions().getObserve();
payloadBytes = response.getPayload();
responseCode = response.getCode();
latch.countDown();
}
@Override
public void onError() {
log.warn("Command Response Ack Error, No connect");
}
}

118
application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java

@ -0,0 +1,118 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.transport.coap;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapHandler;
import org.eclipse.californium.core.CoapObserveRelation;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.eclipse.californium.core.coap.Request;
import org.eclipse.californium.elements.exception.ConnectorException;
import org.thingsboard.server.common.msg.session.FeatureType;
import java.io.IOException;
public class CoapTestClient {
private static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/";
private static final long CLIENT_REQUEST_TIMEOUT = 60000L;
private final CoapClient client;
public CoapTestClient(){
this.client = createClient();
}
public CoapTestClient(String accessToken, FeatureType featureType) {
this.client = createClient(getFeatureTokenUrl(accessToken, featureType));
}
public CoapTestClient(String featureTokenUrl) {
this.client = createClient(featureTokenUrl);
}
public void connectToCoap(String accessToken) {
setURI(accessToken, null);
}
public void connectToCoap(String accessToken, FeatureType featureType) {
setURI(accessToken, featureType);
}
public void disconnect() {
if (client != null) {
client.shutdown();
}
}
public CoapResponse postMethod(String requestBody) throws ConnectorException, IOException {
return this.postMethod(requestBody.getBytes());
}
public CoapResponse postMethod(byte[] requestBodyBytes) throws ConnectorException, IOException {
return client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(requestBodyBytes, MediaTypeRegistry.APPLICATION_JSON);
}
public void postMethod(CoapHandler handler, String payload, int format) {
client.post(handler, payload, format);
}
public void postMethod(CoapHandler handler, byte[] payload, int format) {
client.post(handler, payload, format);
}
public CoapResponse getMethod() throws ConnectorException, IOException {
return client.setTimeout(CLIENT_REQUEST_TIMEOUT).get();
}
public CoapObserveRelation getObserveRelation(CoapTestCallback callback){
Request request = Request.newGet().setObserve();
request.setType(CoAP.Type.CON);
return client.observe(request, callback);
}
public void setURI(String featureTokenUrl) {
if (client == null) {
throw new RuntimeException("Failed to connect! CoapClient is not initialized!");
}
client.setURI(featureTokenUrl);
}
public void setURI(String accessToken, FeatureType featureType) {
if (featureType == null){
featureType = FeatureType.ATTRIBUTES;
}
setURI(getFeatureTokenUrl(accessToken, featureType));
}
private CoapClient createClient() {
return new CoapClient();
}
private CoapClient createClient(String featureTokenUrl) {
return new CoapClient(featureTokenUrl);
}
public static String getFeatureTokenUrl(String token, FeatureType featureType) {
return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase();
}
public static String getFeatureTokenUrl(String token, FeatureType featureType, int requestId) {
return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase() + "/" + requestId;
}
}

376
application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java

@ -15,25 +15,94 @@
*/
package org.thingsboard.server.transport.coap.attributes;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.eclipse.californium.core.CoapObserveRelation;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import org.thingsboard.server.common.data.query.EntityKey;
import org.thingsboard.server.common.data.query.EntityKeyType;
import org.thingsboard.server.common.data.query.SingleEntityFilter;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.common.transport.service.DefaultTransportService;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest;
import org.thingsboard.server.transport.coap.CoapTestCallback;
import org.thingsboard.server.transport.coap.CoapTestClient;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import static org.thingsboard.server.common.data.query.EntityKeyType.CLIENT_ATTRIBUTE;
import static org.thingsboard.server.common.data.query.EntityKeyType.SHARED_ATTRIBUTE;
@Slf4j
public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoapIntegrationTest {
protected static final String POST_ATTRIBUTES_PAYLOAD = "{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73," +
"\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}";
@Autowired
DefaultTransportService defaultTransportService;
public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" +
"\n" +
"package test;\n" +
"\n" +
"message PostAttributes {\n" +
" string clientStr = 1;\n" +
" bool clientBool = 2;\n" +
" double clientDbl = 3;\n" +
" int32 clientLong = 4;\n" +
" JsonObject clientJson = 5;\n" +
"\n" +
" message JsonObject {\n" +
" int32 someNumber = 6;\n" +
" repeated int32 someArray = 7;\n" +
" NestedJsonObject someNestedObject = 8;\n" +
" message NestedJsonObject {\n" +
" string key = 9;\n" +
" }\n" +
" }\n" +
"}";
private static final String CLIENT_ATTRIBUTES_PAYLOAD = "{\"clientStr\":\"value1\",\"clientBool\":true,\"clientDbl\":42.0,\"clientLong\":73," +
"\"clientJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}";
private static final String SHARED_ATTRIBUTES_PAYLOAD = "{\"sharedStr\":\"value1\",\"sharedBool\":true,\"sharedDbl\":42.0,\"sharedLong\":73," +
"\"sharedJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}";
protected static final String SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"sharedStr\":\"value\",\"sharedBool\":false,\"sharedDbl\":41.0,\"sharedLong\":72," +
"\"sharedJson\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}";
protected List<TransportProtos.TsKvProto> getTsKvProtoList() {
TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value1", TransportProtos.KeyValueType.STRING_V);
TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "true", TransportProtos.KeyValueType.BOOLEAN_V);
TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "42.0", TransportProtos.KeyValueType.DOUBLE_V);
TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "73", TransportProtos.KeyValueType.LONG_V);
TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V);
private static final String SHARED_ATTRIBUTES_DELETED_RESPONSE = "{\"deleted\":[\"sharedJson\"]}";
private List<TransportProtos.TsKvProto> getTsKvProtoList(String attributePrefix) {
TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto(attributePrefix + "Str", "value1", TransportProtos.KeyValueType.STRING_V);
TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto(attributePrefix + "Bool", "true", TransportProtos.KeyValueType.BOOLEAN_V);
TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto(attributePrefix + "Dbl", "42.0", TransportProtos.KeyValueType.DOUBLE_V);
TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto(attributePrefix + "Long", "73", TransportProtos.KeyValueType.LONG_V);
TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto(attributePrefix + "Json", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V);
List<TransportProtos.TsKvProto> tsKvProtoList = new ArrayList<>();
tsKvProtoList.add(tsKvProtoAttribute1);
tsKvProtoList.add(tsKvProtoAttribute2);
@ -49,4 +118,295 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap
tsKvProtoBuilder.setKv(keyValueProto);
return tsKvProtoBuilder.build();
}
private List<EntityKey> getEntityKeys(List<String> keys, EntityKeyType scope) {
return keys.stream().map(key -> new EntityKey(scope, key)).collect(Collectors.toList());
}
private byte[] getAttributesProtoPayloadBytes() {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR);
DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
assertNotNull(nestedJsonObjectBuilderDescriptor);
DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build();
DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject");
Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType();
assertNotNull(jsonObjectBuilderDescriptor);
DynamicMessage jsonObject = jsonObjectBuilder
.setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3)
.setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject)
.build();
DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes");
Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType();
assertNotNull(postAttributesMsgDescriptor);
DynamicMessage postAttributesMsg = postAttributesBuilder
.setField(postAttributesMsgDescriptor.findFieldByName("clientStr"), "value1")
.setField(postAttributesMsgDescriptor.findFieldByName("clientBool"), true)
.setField(postAttributesMsgDescriptor.findFieldByName("clientDbl"), 42.0)
.setField(postAttributesMsgDescriptor.findFieldByName("clientLong"), 73)
.setField(postAttributesMsgDescriptor.findFieldByName("clientJson"), jsonObject)
.build();
return postAttributesMsg.toByteArray();
}
protected void processJsonTestRequestAttributesValuesFromTheServer() throws Exception {
client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES);
SingleEntityFilter dtf = new SingleEntityFilter();
dtf.setSingleEntity(savedDevice.getId());
String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson";
String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson";
List<String> clientKeysList = List.of(clientKeysStr.split(","));
List<String> sharedKeysList = List.of(sharedKeysStr.split(","));
List<EntityKey> csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE);
List<EntityKey> shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE);
List<EntityKey> keys = new ArrayList<>();
keys.addAll(csKeys);
keys.addAll(shKeys);
getWsClient().subscribeLatestUpdate(keys, dtf);
getWsClient().registerWaitForUpdate(2);
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE",
SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
CoapResponse coapResponse = client.postMethod(CLIENT_ATTRIBUTES_PAYLOAD);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
String update = getWsClient().waitForUpdate();
assertThat(update).as("ws update received").isNotBlank();
String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr;
client.setURI(featureTokenUrl);
validateJsonResponse(client.getMethod());
}
protected void processProtoTestRequestAttributesValuesFromTheServer() throws Exception {
client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES);
SingleEntityFilter dtf = new SingleEntityFilter();
dtf.setSingleEntity(savedDevice.getId());
String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson";
String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson";
List<String> clientKeysList = List.of(clientKeysStr.split(","));
List<String> sharedKeysList = List.of(sharedKeysStr.split(","));
List<EntityKey> csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE);
List<EntityKey> shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE);
List<EntityKey> keys = new ArrayList<>();
keys.addAll(csKeys);
keys.addAll(shKeys);
getWsClient().subscribeLatestUpdate(keys, dtf);
getWsClient().registerWaitForUpdate(2);
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE",
SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
CoapResponse coapResponse = client.postMethod(getAttributesProtoPayloadBytes());
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
String update = getWsClient().waitForUpdate();
assertThat(update).as("ws update received").isNotBlank();
String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr;
client.setURI(featureTokenUrl);
validateProtoResponse(client.getMethod());
}
protected void processJsonTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception {
if (!emptyCurrentStateNotification) {
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk());
}
client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES);
CoapTestCallback callbackCoap = new CoapTestCallback(1);
CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap);
callbackCoap.getLatch().await(3, TimeUnit.SECONDS);
if (emptyCurrentStateNotification) {
validateUpdateAttributesJsonResponse(callbackCoap, "{}", 0);
} else {
validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, 0);
}
CountDownLatch latch = new CountDownLatch(1);
int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1;
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
latch.await(3, TimeUnit.SECONDS);
validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD, expectedObserveCnt);
latch = new CountDownLatch(1);
int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1;
doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class);
latch.await(3, TimeUnit.SECONDS);
validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_DELETED_RESPONSE, expectedObserveBeforeDeleteCnt);
observeRelation.proactiveCancel();
assertTrue(observeRelation.isCanceled());
awaitClientAfterCancelObserve();
}
protected void processProtoTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception {
if (!emptyCurrentStateNotification) {
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk());
}
client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES);
CoapTestCallback callbackCoap = new CoapTestCallback(1);
CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap);
callbackCoap.getLatch().await(3, TimeUnit.SECONDS);
if (emptyCurrentStateNotification) {
validateEmptyCurrentStateAttributesProtoResponse(callbackCoap);
} else {
validateCurrentStateAttributesProtoResponse(callbackCoap);
}
CountDownLatch latch = new CountDownLatch(1);
int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1;
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
latch.await(3, TimeUnit.SECONDS);
validateUpdateProtoAttributesResponse(callbackCoap, expectedObserveCnt);
latch = new CountDownLatch(1);
int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1;
doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class);
latch.await(3, TimeUnit.SECONDS);
validateDeleteProtoAttributesResponse(callbackCoap, expectedObserveBeforeDeleteCnt);
observeRelation.proactiveCancel();
assertTrue(observeRelation.isCanceled());
awaitClientAfterCancelObserve();
}
protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException {
assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode());
String expectedResponse = "{\"client\":" + CLIENT_ATTRIBUTES_PAYLOAD + ",\"shared\":" + SHARED_ATTRIBUTES_PAYLOAD + "}";
assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(getAttributesResponse.getPayload()));
}
protected void validateProtoResponse(CoapResponse getAttributesResponse) throws InterruptedException, InvalidProtocolBufferException {
TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg();
TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload());
assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId());
List<TransportProtos.KeyValueProto> expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos));
assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos));
}
protected void validateUpdateAttributesJsonResponse(CoapTestCallback callback, String expectedResponse, int expectedObserveCnt) {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveCnt, callback.getObserve().intValue());
String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8);
assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.toJsonNode(response));
}
protected void validateEmptyCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException {
assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(0, callback.getObserve().intValue());
}
protected void validateCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(0, callback.getObserve().intValue());
TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("sharedStr", "value", TransportProtos.KeyValueType.STRING_V);
TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("sharedBool", "false", TransportProtos.KeyValueType.BOOLEAN_V);
TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("sharedDbl", "41.0", TransportProtos.KeyValueType.DOUBLE_V);
TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("sharedLong", "72", TransportProtos.KeyValueType.LONG_V);
TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("sharedJson", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V);
List<TransportProtos.TsKvProto> tsKvProtoList = new ArrayList<>();
tsKvProtoList.add(tsKvProtoAttribute1);
tsKvProtoList.add(tsKvProtoAttribute2);
tsKvProtoList.add(tsKvProtoAttribute3);
tsKvProtoList.add(tsKvProtoAttribute4);
tsKvProtoList.add(tsKvProtoAttribute5);
TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build();
TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes());
List<TransportProtos.KeyValueProto> expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size());
assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList));
}
protected void validateUpdateProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveCnt, callback.getObserve().intValue());
TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
List<TransportProtos.TsKvProto> tsKvProtoList = getTsKvProtoList("shared");
attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList);
TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build();
TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes());
List<TransportProtos.KeyValueProto> actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size());
assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList));
}
protected void validateDeleteProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveCnt, callback.getObserve().intValue());
TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson");
TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build();
TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes());
assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size());
assertEquals("sharedJson", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0));
}
private void awaitClientAfterCancelObserve() {
Awaitility.await("awaitClientAfterCancelObserve")
.pollInterval(10, TimeUnit.MILLISECONDS)
.atMost(5, TimeUnit.SECONDS)
.until(()->{
log.trace("awaiting defaultTransportService.sessions is empty");
return defaultTransportService.sessions.isEmpty();});
}
private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() {
TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder();
List<TransportProtos.TsKvProto> csTsKvProtoList = getTsKvProtoList("client");
List<TransportProtos.TsKvProto> shTsKvProtoList = getTsKvProtoList("shared");
result.addAllClientAttributeList(csTsKvProtoList);
result.addAllSharedAttributeList(shTsKvProtoList);
result.setRequestId(0);
return result.build();
}
}

56
application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java

@ -15,34 +15,18 @@
*/
package org.thingsboard.server.transport.coap.attributes.request;
import com.fasterxml.jackson.core.type.TypeReference;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest;
import org.thingsboard.server.common.msg.session.FeatureType;
import java.nio.charset.StandardCharsets;
import java.util.List;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@Slf4j
@DaoSqlTest
public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributesIntegrationTest {
protected static final long CLIENT_REQUEST_TIMEOUT = 60000L;
@Before
public void beforeTest() throws Exception {
CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder()
@ -58,44 +42,6 @@ public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributes
@Test
public void testRequestAttributesValuesFromTheServer() throws Exception {
processTestRequestAttributesValuesFromTheServer();
}
protected void processTestRequestAttributesValuesFromTheServer() throws Exception {
postAttributes();
long start = System.currentTimeMillis();
long end = System.currentTimeMillis() + 5000;
List<String> savedAttributeKeys = null;
while (start <= end) {
savedAttributeKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {});
if (savedAttributeKeys.size() == 5) {
break;
}
Thread.sleep(100);
start += 100;
}
assertNotNull(savedAttributeKeys);
String keys = "attribute1,attribute2,attribute3,attribute4,attribute5";
String featureTokenUrl = getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys;
client = getCoapClient(featureTokenUrl);
CoapResponse getAttributesResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).get();
validateResponse(getAttributesResponse);
}
protected void postAttributes() throws Exception {
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
client = getCoapClient(FeatureType.ATTRIBUTES);
CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
}
protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException {
assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode());
String expectedRequestPayload = "{\"client\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}},\"shared\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}";
assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(getAttributesResponse.getPayload(), StandardCharsets.UTF_8)));
processJsonTestRequestAttributesValuesFromTheServer();
}
}

2
application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java

@ -45,6 +45,6 @@ public class CoapAttributesRequestJsonIntegrationTest extends CoapAttributesRequ
@Test
public void testRequestAttributesValuesFromTheServer() throws Exception {
super.testRequestAttributesValuesFromTheServer();
processJsonTestRequestAttributesValuesFromTheServer();
}
}

116
application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java

@ -15,64 +15,18 @@
*/
package org.thingsboard.server.transport.coap.attributes.request;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@Slf4j
@DaoSqlTest
public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesRequestIntegrationTest {
public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" +
"\n" +
"package test;\n" +
"\n" +
"message PostAttributes {\n" +
" string attribute1 = 1;\n" +
" bool attribute2 = 2;\n" +
" double attribute3 = 3;\n" +
" int32 attribute4 = 4;\n" +
" JsonObject attribute5 = 5;\n" +
"\n" +
" message JsonObject {\n" +
" int32 someNumber = 6;\n" +
" repeated int32 someArray = 7;\n" +
" NestedJsonObject someNestedObject = 8;\n" +
" message NestedJsonObject {\n" +
" string key = 9;\n" +
" }\n" +
" }\n" +
"}";
@Before
@Override
public void beforeTest() throws Exception {
@ -87,74 +41,6 @@ public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesReq
@Test
public void testRequestAttributesValuesFromTheServer() throws Exception {
processTestRequestAttributesValuesFromTheServer();
processProtoTestRequestAttributesValuesFromTheServer();
}
protected void postAttributes() throws Exception {
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR);
DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA);
DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject");
Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType();
assertNotNull(nestedJsonObjectBuilderDescriptor);
DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build();
DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject");
Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType();
assertNotNull(jsonObjectBuilderDescriptor);
DynamicMessage jsonObject = jsonObjectBuilder
.setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2)
.addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3)
.setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject)
.build();
DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes");
Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType();
assertNotNull(postAttributesMsgDescriptor);
DynamicMessage postAttributesMsg = postAttributesBuilder
.setField(postAttributesMsgDescriptor.findFieldByName("attribute1"), "value1")
.setField(postAttributesMsgDescriptor.findFieldByName("attribute2"), true)
.setField(postAttributesMsgDescriptor.findFieldByName("attribute3"), 42.0)
.setField(postAttributesMsgDescriptor.findFieldByName("attribute4"), 73)
.setField(postAttributesMsgDescriptor.findFieldByName("attribute5"), jsonObject)
.build();
byte[] payload = postAttributesMsg.toByteArray();
client = getCoapClient(FeatureType.ATTRIBUTES);
CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(payload, MediaTypeRegistry.APPLICATION_JSON);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
}
protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException {
TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg();
TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload());
assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId());
List<TransportProtos.KeyValueProto> expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos));
assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos));
}
private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() {
TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder();
List<TransportProtos.TsKvProto> tsKvProtoList = getTsKvProtoList();
result.addAllClientAttributeList(tsKvProtoList);
result.addAllSharedAttributeList(tsKvProtoList);
result.setRequestId(0);
return result.build();
}
}

151
application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java

@ -15,47 +15,25 @@
*/
package org.thingsboard.server.transport.coap.attributes.updates;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.awaitility.Awaitility;
import org.eclipse.californium.core.CoapHandler;
import org.eclipse.californium.core.CoapObserveRelation;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.Request;
import org.eclipse.californium.core.server.resources.Resource;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.coapserver.DefaultCoapServerService;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.common.transport.service.DefaultTransportService;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import org.thingsboard.server.transport.coap.CoapTransportResource;
import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.spy;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@Slf4j
@DaoSqlTest
public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributesIntegrationTest {
private static final String RESPONSE_ATTRIBUTES_PAYLOAD_DELETED = "{\"deleted\":[\"attribute5\"]}";
protected static final String POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"attribute1\":\"value\",\"attribute2\":false,\"attribute3\":41.0,\"attribute4\":72," +
"\"attribute5\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}";
CoapTransportResource coapTransportResource;
@Autowired
@ -83,136 +61,11 @@ public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributes
@Test
public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception {
processTestSubscribeToAttributesUpdates(false);
processJsonTestSubscribeToAttributesUpdates(false);
}
@Test
public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception {
processTestSubscribeToAttributesUpdates(true);
}
protected void processTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception {
if (!emptyCurrentStateNotification) {
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk());
}
client = getCoapClient(FeatureType.ATTRIBUTES);
CountDownLatch latch = new CountDownLatch(1);
TestCoapCallback callback = new TestCoapCallback(latch);
Request request = Request.newGet().setObserve();
request.setType(CoAP.Type.CON);
CoapObserveRelation observeRelation = client.observe(request, callback);
latch.await(3, TimeUnit.SECONDS);
if (emptyCurrentStateNotification) {
validateEmptyCurrentStateAttributesResponse(callback);
} else {
validateCurrentStateAttributesResponse(callback);
}
latch = new CountDownLatch(1);
int expectedObserveCnt = callback.getObserve().intValue() + 1;
doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk());
latch.await(3, TimeUnit.SECONDS);
validateUpdateAttributesResponse(callback, expectedObserveCnt);
latch = new CountDownLatch(1);
int expectedObserveBeforeDeleteCnt = callback.getObserve().intValue() + 1;
doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=attribute5", String.class);
latch.await(3, TimeUnit.SECONDS);
validateDeleteAttributesResponse(callback, expectedObserveBeforeDeleteCnt);
observeRelation.proactiveCancel();
assertTrue(observeRelation.isCanceled());
awaitClientAfterCancelObserve();
}
protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(0, callback.getObserve().intValue());
String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8);
assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION), JacksonUtil.toJsonNode(response));
}
protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(0, callback.getObserve().intValue());
String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8);
assertEquals("{}", response);
}
protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveCnt, callback.getObserve().intValue());
String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8);
assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD), JacksonUtil.toJsonNode(response));
}
protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveCnt, callback.getObserve().intValue());
String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8);
assertEquals(JacksonUtil.toJsonNode(RESPONSE_ATTRIBUTES_PAYLOAD_DELETED), JacksonUtil.toJsonNode(response));
}
protected static class TestCoapCallback implements CoapHandler {
private final CountDownLatch latch;
private Integer observe;
private byte[] payloadBytes;
private CoAP.ResponseCode responseCode;
public Integer getObserve() {
return observe;
}
public byte[] getPayloadBytes() {
return payloadBytes;
}
public CoAP.ResponseCode getResponseCode() {
return responseCode;
}
private TestCoapCallback(CountDownLatch latch) {
this.latch = latch;
}
@Override
public void onLoad(CoapResponse response) {
observe = response.getOptions().getObserve();
payloadBytes = response.getPayload();
responseCode = response.getCode();
latch.countDown();
}
@Override
public void onError() {
log.warn("Command Response Ack Error, No connect");
}
}
private void awaitClientAfterCancelObserve() {
Awaitility.await("awaitClientAfterCancelObserve")
.pollInterval(10, TimeUnit.MILLISECONDS)
.atMost(5, TimeUnit.SECONDS)
.until(()->{
log.trace("awaiting defaultTransportService.sessions is empty");
return defaultTransportService.sessions.isEmpty();});
processJsonTestSubscribeToAttributesUpdates(true);
}
}

7
application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java

@ -23,10 +23,11 @@ import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest;
@Slf4j
@DaoSqlTest
public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpdatesIntegrationTest {
public class CoapAttributesUpdatesJsonIntegrationTest extends AbstractCoapAttributesIntegrationTest {
@Before
public void beforeTest() throws Exception {
@ -45,11 +46,11 @@ public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpda
@Test
public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception {
super.testSubscribeToAttributesUpdatesFromTheServer();
processJsonTestSubscribeToAttributesUpdates(false);
}
@Test
public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception {
super.testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification();
processJsonTestSubscribeToAttributesUpdates(true);
}
}

91
application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java

@ -15,30 +15,19 @@
*/
package org.thingsboard.server.transport.coap.attributes.updates;
import com.google.protobuf.InvalidProtocolBufferException;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.coap.CoAP;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertTrue;
import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest;
@Slf4j
@DaoSqlTest
public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpdatesIntegrationTest {
public class CoapAttributesUpdatesProtoIntegrationTest extends AbstractCoapAttributesIntegrationTest {
@Before
public void beforeTest() throws Exception {
@ -57,83 +46,11 @@ public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpd
@Test
public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception {
processTestSubscribeToAttributesUpdates(false);
processProtoTestSubscribeToAttributesUpdates(false);
}
@Test
public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception {
processTestSubscribeToAttributesUpdates(true);
processProtoTestSubscribeToAttributesUpdates(true);
}
protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(0, callback.getObserve().intValue());
TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value", TransportProtos.KeyValueType.STRING_V);
TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "false", TransportProtos.KeyValueType.BOOLEAN_V);
TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "41.0", TransportProtos.KeyValueType.DOUBLE_V);
TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "72", TransportProtos.KeyValueType.LONG_V);
TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V);
List<TransportProtos.TsKvProto> tsKvProtoList = new ArrayList<>();
tsKvProtoList.add(tsKvProtoAttribute1);
tsKvProtoList.add(tsKvProtoAttribute2);
tsKvProtoList.add(tsKvProtoAttribute3);
tsKvProtoList.add(tsKvProtoAttribute4);
tsKvProtoList.add(tsKvProtoAttribute5);
TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build();
TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes());
List<TransportProtos.KeyValueProto> expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size());
assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList));
}
protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException {
assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(0, callback.getObserve().intValue());
}
protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveCnt, callback.getObserve().intValue());
TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
List<TransportProtos.TsKvProto> tsKvProtoList = getTsKvProtoList();
attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList);
TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build();
TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes());
List<TransportProtos.KeyValueProto> actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
List<TransportProtos.KeyValueProto> expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList());
assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size());
assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList));
}
protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException {
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveCnt, callback.getObserve().intValue());
TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder();
attributeUpdateNotificationMsgBuilder.addSharedDeleted("attribute5");
TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build();
TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes());
assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size());
assertEquals("attribute5", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0));
}
}

11
application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java

@ -16,10 +16,8 @@
package org.thingsboard.server.transport.coap.claim;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.eclipse.californium.elements.exception.ConnectorException;
import org.junit.After;
import org.junit.Before;
@ -34,6 +32,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse;
import org.thingsboard.server.dao.device.claim.ClaimResult;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest;
import org.thingsboard.server.transport.coap.CoapTestClient;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.io.IOException;
@ -96,7 +95,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest {
protected void processTestClaimingDevice(boolean emptyPayload) throws Exception {
log.warn("[testClaimingDevice] Device: {}, Transport type: {}", savedDevice.getName(), savedDevice.getType());
client = getCoapClient(FeatureType.CLAIM);
client = new CoapTestClient(accessToken, FeatureType.CLAIM);
byte[] payloadBytes;
byte[] failurePayloadBytes;
if (emptyPayload) {
@ -109,7 +108,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest {
validateClaimResponse(emptyPayload, client, payloadBytes, failurePayloadBytes);
}
protected void validateClaimResponse(boolean emptyPayload, CoapClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception {
protected void validateClaimResponse(boolean emptyPayload, CoapTestClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception {
postClaimRequest(client, failurePayloadBytes);
loginUser(customerAdmin.getName(), CUSTOMER_USER_PASSWORD);
@ -145,8 +144,8 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest {
assertEquals(claimResponse, ClaimResponse.CLAIMED);
}
private void postClaimRequest(CoapClient client, byte[] payload) throws IOException, ConnectorException {
CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON);
private void postClaimRequest(CoapTestClient client, byte[] payload) throws IOException, ConnectorException {
CoapResponse coapResponse = client.postMethod(payload);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
}

3
application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java

@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.gen.transport.TransportApiProtos;
import org.thingsboard.server.transport.coap.CoapTestClient;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
@Slf4j
@ -56,7 +57,7 @@ public class CoapClaimProtoDeviceTest extends CoapClaimDeviceTest {
@Override
protected void processTestClaimingDevice(boolean emptyPayload) throws Exception {
client = getCoapClient(FeatureType.CLAIM);
client = new CoapTestClient(accessToken, FeatureType.CLAIM);
byte[] payloadBytes;
if (emptyPayload) {
TransportApiProtos.ClaimDevice claimDevice = getClaimDevice(0, emptyPayload);

78
application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java

@ -15,16 +15,13 @@
*/
package org.thingsboard.server.transport.coap.provision;
import com.google.gson.JsonObject;
import com.fasterxml.jackson.databind.JsonNode;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.eclipse.californium.elements.exception.ConnectorException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest;
import org.thingsboard.server.common.data.CoapDeviceType;
@ -34,14 +31,12 @@ import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.security.DeviceCredentials;
import org.thingsboard.server.common.msg.EncryptionUtil;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.common.transport.util.JsonUtils;
import org.thingsboard.server.dao.device.DeviceCredentialsService;
import org.thingsboard.server.dao.device.DeviceService;
import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus;
import org.thingsboard.server.transport.coap.CoapTestClient;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.io.IOException;
@Slf4j
@DaoSqlTest
public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
@ -95,10 +90,11 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
.transportPayloadType(TransportPayloadType.JSON)
.build();
processBeforeTest(configProperties);
byte[] result = createCoapClientAndPublish().getPayload();
JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject();
Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString());
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString());
JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish());
Assert.assertTrue(response.hasNonNull("errorMsg"));
Assert.assertTrue(response.hasNonNull("status"));
Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText());
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText());
}
@ -112,8 +108,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
byte[] result = createCoapClientAndPublish().getPayload();
JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject();
JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish());
Assert.assertTrue(response.hasNonNull("credentialsType"));
Assert.assertTrue(response.hasNonNull("status"));
Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device");
@ -121,8 +118,8 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText());
}
@ -137,8 +134,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
.build();
processBeforeTest(configProperties);
String requestCredentials = ",\"credentialsType\": \"ACCESS_TOKEN\",\"token\": \"test_token\"";
byte[] result = createCoapClientAndPublish(requestCredentials).getPayload();
JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject();
JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials));
Assert.assertTrue(response.hasNonNull("credentialsType"));
Assert.assertTrue(response.hasNonNull("status"));
Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device");
@ -146,10 +144,10 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "ACCESS_TOKEN");
Assert.assertEquals(deviceCredentials.getCredentialsId(), "test_token");
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText());
}
@ -164,8 +162,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
.build();
processBeforeTest(configProperties);
String requestCredentials = ",\"credentialsType\": \"X509_CERTIFICATE\",\"hash\": \"testHash\"";
byte[] result = createCoapClientAndPublish(requestCredentials).getPayload();
JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject();
JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials));
Assert.assertTrue(response.hasNonNull("credentialsType"));
Assert.assertTrue(response.hasNonNull("status"));
Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device");
@ -173,7 +172,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "X509_CERTIFICATE");
String cert = EncryptionUtil.certTrimNewLines(deviceCredentials.getCredentialsValue());
@ -182,7 +181,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
Assert.assertEquals(deviceCredentials.getCredentialsId(), sha3Hash);
Assert.assertEquals(deviceCredentials.getCredentialsValue(), "testHash");
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText());
}
private void processTestProvisioningCheckPreProvisionedDevice() throws Exception {
@ -195,13 +194,14 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
byte[] result = createCoapClientAndPublish().getPayload();
JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject();
JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish());
Assert.assertTrue(response.hasNonNull("credentialsType"));
Assert.assertTrue(response.hasNonNull("status"));
DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText());
}
private void processTestProvisioningWithBadKeyDevice() throws Exception {
@ -214,25 +214,21 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
byte[] result = createCoapClientAndPublish().getPayload();
JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject();
Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString());
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString());
JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish());
Assert.assertTrue(response.hasNonNull("errorMsg"));
Assert.assertTrue(response.hasNonNull("status"));
Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText());
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText());
}
private CoapResponse createCoapClientAndPublish() throws Exception {
private byte[] createCoapClientAndPublish() throws Exception {
return createCoapClientAndPublish("");
}
private CoapResponse createCoapClientAndPublish(String deviceCredentials) throws Exception {
private byte[] createCoapClientAndPublish(String deviceCredentials) throws Exception {
String provisionRequestMsg = createTestProvisionMessage(deviceCredentials);
client = getCoapClient(FeatureType.PROVISION);
return postProvision(client, provisionRequestMsg.getBytes());
}
private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException {
return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON);
client = new CoapTestClient(accessToken, FeatureType.PROVISION);
return client.postMethod(provisionRequestMsg.getBytes()).getPayload();
}
private String createTestProvisionMessage(String deviceCredentials) {

61
application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java

@ -16,10 +16,7 @@
package org.thingsboard.server.transport.coap.provision;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.eclipse.californium.elements.exception.ConnectorException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@ -44,10 +41,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceReque
import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg;
import org.thingsboard.server.transport.coap.CoapTestClient;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.io.IOException;
@Slf4j
@DaoSqlTest
public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
@ -101,9 +97,9 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
.transportPayloadType(TransportPayloadType.PROTOBUF)
.build();
processBeforeTest(configProperties);
ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload());
ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish());
Assert.assertNotNull(result);
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().toString());
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().name());
}
private void processTestProvisioningCreateNewDeviceWithoutCredentials() throws Exception {
@ -116,7 +112,7 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload());
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish());
Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device");
@ -124,8 +120,8 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name());
}
private void processTestProvisioningCreateNewDeviceWithAccessToken() throws Exception {
@ -138,9 +134,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()).build();
CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder()
.setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build())
.build();
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials)).getPayload());
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(
createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials)));
Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device");
@ -164,9 +163,13 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceX509CertRequestMsg(ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()).build();
CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder()
.setValidateDeviceX509CertRequestMsg(
ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build())
.build();
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials)).getPayload());
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(
createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials)));
Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device");
@ -196,12 +199,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload());
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish());
DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString());
Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name());
Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name());
}
private void processTestProvisioningWithBadKeyDevice() throws Exception {
@ -214,24 +217,19 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
.provisionSecret("testProvisionSecret")
.build();
processBeforeTest(configProperties);
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload());
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().toString());
}
private CoapResponse createCoapClientAndPublish() throws Exception {
byte[] provisionRequestMsg = createTestProvisionMessage();
CoapResponse coapResponse = createCoapClientAndPublish(provisionRequestMsg);
Assert.assertNotNull("COAP response", coapResponse);
return coapResponse;
ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish());
Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().name());
}
private CoapResponse createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception {
client = getCoapClient(FeatureType.PROVISION);
return postProvision(client, provisionRequestMsg);
private byte[] createCoapClientAndPublish() throws Exception {
return createCoapClientAndPublish(createTestProvisionMessage());
}
private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException {
return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON);
private byte[] createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception {
client = new CoapTestClient(accessToken, FeatureType.PROVISION);
CoapResponse coapResponse = client.postMethod(provisionRequestMsg);
Assert.assertNotNull("COAP response", coapResponse);
return coapResponse.getPayload();
}
private byte[] createTestsProvisionMessage(CredentialsType credentialsType, CredentialsDataProto credentialsData) throws Exception {
@ -247,7 +245,6 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest {
.toByteArray();
}
private byte[] createTestProvisionMessage() throws Exception {
return createTestsProvisionMessage(null, null);
}

214
application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java

@ -16,18 +16,29 @@
package org.thingsboard.server.transport.coap.rpc;
import com.fasterxml.jackson.databind.JsonNode;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapHandler;
import org.eclipse.californium.core.CoapObserveRelation;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.eclipse.californium.core.coap.Request;
import org.thingsboard.common.util.JacksonUtil;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest;
import org.thingsboard.server.transport.coap.CoapTestCallback;
import org.thingsboard.server.transport.coap.CoapTestClient;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@ -41,76 +52,74 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
@Slf4j
public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractCoapIntegrationTest {
public static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" +
"package rpc;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string method = 1;\n" +
" optional int32 requestId = 2;\n" +
" Params params = 3;\n" +
"\n" +
" message Params {\n" +
" optional string pin = 1;\n" +
" optional int32 value = 2;\n" +
" }\n" +
"}";
protected static final String DEVICE_RESPONSE = "{\"value1\":\"A\",\"value2\":\"B\"}";
protected static final Long asyncContextTimeoutToUseRpcPlugin = 10000L;
protected void processOneWayRpcTest() throws Exception {
client = getCoapClient(FeatureType.RPC);
client.useCONs();
CountDownLatch latch = new CountDownLatch(1);
TestCoapCallback callback = new TestCoapCallback(client, latch, true);
protected void processOneWayRpcTest(boolean protobuf) throws Exception {
client = new CoapTestClient(accessToken, FeatureType.RPC);
CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, true, protobuf);
Request request = Request.newGet().setObserve();
CoapObserveRelation observeRelation = client.observe(request, callback);
latch.await(3, TimeUnit.SECONDS);
validateCurrentStateNotification(callback);
latch = new CountDownLatch(1);
CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap);
callbackCoap.getLatch().await(3, TimeUnit.SECONDS);
validateCurrentStateNotification(callbackCoap);
CountDownLatch latch = new CountDownLatch(1);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}";
String deviceId = savedDevice.getId().getId().toString();
String result = doPostAsync("/api/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk());
latch.await(3, TimeUnit.SECONDS);
validateOneWayStateChangedNotification(callback, result);
validateOneWayStateChangedNotification(callbackCoap, result);
observeRelation.proactiveCancel();
assertTrue(observeRelation.isCanceled());
}
protected void processTwoWayRpcTest(String expectedResponseResult) throws Exception {
client = getCoapClient(FeatureType.RPC);
client.useCONs();
CountDownLatch latch = new CountDownLatch(1);
TestCoapCallback callback = new TestCoapCallback(client, latch, false);
protected void processTwoWayRpcTest(String expectedResponseResult, boolean protobuf) throws Exception {
client = new CoapTestClient(accessToken, FeatureType.RPC);
CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, false, protobuf);
Request request = Request.newGet().setObserve();
request.setType(CoAP.Type.CON);
CoapObserveRelation observeRelation = client.observe(request, callback);
latch.await(3, TimeUnit.SECONDS);
CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap);
callbackCoap.getLatch().await(3, TimeUnit.SECONDS);
validateCurrentStateNotification(callback);
validateCurrentStateNotification(callbackCoap);
String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}";
String deviceId = savedDevice.getId().getId().toString();
String actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk());
latch.await(3, TimeUnit.SECONDS);
validateTwoWayStateChangedNotification(callback, 1, expectedResponseResult, actualResult);
callbackCoap.getLatch().await(3, TimeUnit.SECONDS);
latch = new CountDownLatch(1);
validateTwoWayStateChangedNotification(callbackCoap, 1, expectedResponseResult, actualResult);
CountDownLatch latch = new CountDownLatch(1);
actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk());
latch.await(3, TimeUnit.SECONDS);
callbackCoap.getLatch().await(3, TimeUnit.SECONDS);
validateTwoWayStateChangedNotification(callback, 2, expectedResponseResult, actualResult);
validateTwoWayStateChangedNotification(callbackCoap, 2, expectedResponseResult, actualResult);
observeRelation.proactiveCancel();
assertTrue(observeRelation.isCanceled());
}
protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) {
protected void processOnLoadResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) {
JsonNode responseJson = JacksonUtil.fromBytes(response.getPayload());
client.setURI(getRpcResponseFeatureTokenUrl(accessToken, responseJson.get("id").asInt()));
client.post(new CoapHandler() {
client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, responseJson.get("id").asInt()));
client.postMethod(new CoapHandler() {
@Override
public void onLoad(CoapResponse response) {
log.warn("Command Response Ack: {}, {}", response.getCode(), response.getResponseText());
@ -124,36 +133,89 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC
}, DEVICE_RESPONSE, MediaTypeRegistry.APPLICATION_JSON);
}
protected String getRpcResponseFeatureTokenUrl(String token, int requestId) {
return COAP_BASE_URL + token + "/" + FeatureType.RPC.name().toLowerCase() + "/" + requestId;
protected void processOnLoadProtoResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration();
ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA);
DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA);
byte[] requestPayload = response.getPayload();
DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg");
Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType();
try {
DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload);
Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId");
int requestId = (int) dynamicMessage.getField(requestIdDescriptor);
ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA);
DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA);
DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg");
Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType();
DynamicMessage rpcResponseMsg = rpcResponseBuilder
.setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE)
.build();
client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, requestId));
client.postMethod(new CoapHandler() {
@Override
public void onLoad(CoapResponse response) {
log.warn("Command Response Ack: {}", response.getCode());
latch.countDown();
}
@Override
public void onError() {
log.warn("Command Response Ack Error, No connect");
}
}, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON);
} catch (InvalidProtocolBufferException e) {
log.warn("Command Response Ack Error, Invalid response received: ", e);
}
}
protected class TestCoapCallback implements CoapHandler {
private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
}
private final CoapClient client;
private final CountDownLatch latch;
private final boolean isOneWayRpc;
private void validateCurrentStateNotification(CoapTestCallback callback) {
assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID);
assertEquals(0, callback.getObserve().intValue());
}
private Integer observe;
private byte[] payloadBytes;
private CoAP.ResponseCode responseCode;
private void validateOneWayStateChangedNotification(CoapTestCallback callback, String result) {
assertTrue(StringUtils.isEmpty(result));
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(1, callback.getObserve().intValue());
}
public Integer getObserve() {
return observe;
}
private void validateTwoWayStateChangedNotification(CoapTestCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) {
assertEquals(expectedResult, actualResult);
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveNumber, callback.getObserve().intValue());
}
public byte[] getPayloadBytes() {
return payloadBytes;
}
protected class TestCoapCallbackForRPC extends CoapTestCallback {
public CoAP.ResponseCode getResponseCode() {
return responseCode;
}
private final CoapTestClient client;
private final boolean isOneWayRpc;
private final boolean protobuf;
TestCoapCallback(CoapClient client, CountDownLatch latch, boolean isOneWayRpc) {
TestCoapCallbackForRPC(CoapTestClient client, int subscribeCount, boolean isOneWayRpc, boolean protobuf) {
super(subscribeCount);
this.client = client;
this.latch = latch;
this.isOneWayRpc = isOneWayRpc;
this.protobuf = protobuf;
}
@Override
@ -163,7 +225,11 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC
observe = response.getOptions().getObserve();
if (observe != null) {
if (!isOneWayRpc && observe > 0) {
processOnLoadResponse(response, client, observe, latch);
if (!protobuf){
processOnLoadResponse(response, client, observe, latch);
} else {
processOnLoadProtoResponse(response, client, observe, latch);
}
} else {
latch.countDown();
}
@ -174,31 +240,5 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC
public void onError() {
log.warn("Command Response Ack Error, No connect");
}
}
private void validateCurrentStateNotification(TestCoapCallback callback) {
assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID);
assertEquals(0, callback.getObserve().intValue());
}
private void validateOneWayStateChangedNotification(TestCoapCallback callback, String result) {
assertTrue(StringUtils.isEmpty(result));
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(1, callback.getObserve().intValue());
}
private void validateTwoWayStateChangedNotification(TestCoapCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) {
assertEquals(expectedResult, actualResult);
assertNotNull(callback.getPayloadBytes());
assertNotNull(callback.getObserve());
assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode());
assertEquals(expectedObserveNumber, callback.getObserve().intValue());
}
}

7
application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java

@ -21,9 +21,6 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.service.security.AccessValidator;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
@ -87,12 +84,12 @@ public class CoapServerSideRpcDefaultIntegrationTest extends AbstractCoapServerS
@Test
public void testServerCoapOneWayRpc() throws Exception {
processOneWayRpcTest();
processOneWayRpcTest(false);
}
@Test
public void testServerCoapTwoWayRpc() throws Exception {
processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}");
processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false);
}
}

4
application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java

@ -45,12 +45,12 @@ public class CoapServerSideRpcJsonIntegrationTest extends AbstractCoapServerSide
@Test
public void testServerCoapOneWayRpc() throws Exception {
processOneWayRpcTest();
processOneWayRpcTest(false);
}
@Test
public void testServerCoapTwoWayRpc() throws Exception {
processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}");
processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false);
}
}

87
application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java

@ -15,52 +15,19 @@
*/
package org.thingsboard.server.transport.coap.rpc;
import com.github.os72.protobuf.dynamic.DynamicSchema;
import com.google.protobuf.Descriptors;
import com.google.protobuf.DynamicMessage;
import com.google.protobuf.InvalidProtocolBufferException;
import com.squareup.wire.schema.internal.parser.ProtoFileElement;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapHandler;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration;
import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration;
import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration;
import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.util.concurrent.CountDownLatch;
import static org.junit.Assert.assertTrue;
@Slf4j
@DaoSqlTest
public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSideRpcIntegrationTest {
private static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" +
"package rpc;\n" +
"\n" +
"message RpcRequestMsg {\n" +
" optional string method = 1;\n" +
" optional int32 requestId = 2;\n" +
" Params params = 3;\n" +
"\n" +
" message Params {\n" +
" optional string pin = 1;\n" +
" optional int32 value = 2;\n" +
" }\n" +
"}";
@Before
public void beforeTest() throws Exception {
CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder()
@ -79,61 +46,11 @@ public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSid
@Test
public void testServerCoapOneWayRpc() throws Exception {
processOneWayRpcTest();
processOneWayRpcTest(true);
}
@Test
public void testServerCoapTwoWayRpc() throws Exception {
processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}");
}
@Override
protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) {
ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration();
ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA);
DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA);
byte[] requestPayload = response.getPayload();
DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg");
Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType();
try {
DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload);
Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId");
int requestId = (int) dynamicMessage.getField(requestIdDescriptor);
ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA);
DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA);
DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg");
Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType();
DynamicMessage rpcResponseMsg = rpcResponseBuilder
.setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE)
.build();
client.setURI(getRpcResponseFeatureTokenUrl(accessToken, requestId));
client.post(new CoapHandler() {
@Override
public void onLoad(CoapResponse response) {
log.warn("Command Response Ack: {}", response.getCode());
latch.countDown();
}
@Override
public void onError() {
log.warn("Command Response Ack Error, No connect");
}
}, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON);
} catch (InvalidProtocolBufferException e) {
log.warn("Command Response Ack Error, Invalid response received: ", e);
}
}
private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() {
DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration();
assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration);
CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration;
CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration();
assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration);
DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration;
TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration();
assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration);
return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration;
processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}", true);
}
}

50
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java

@ -17,11 +17,8 @@ package org.thingsboard.server.transport.coap.telemetry.attributes;
import com.fasterxml.jackson.core.type.TypeReference;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.eclipse.californium.elements.exception.ConnectorException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@ -29,9 +26,9 @@ import org.thingsboard.server.dao.service.DaoSqlTest;
import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.transport.coap.CoapTestClient;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashMap;
@ -74,30 +71,17 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest {
}
protected void processAttributesTest(List<String> expectedKeys, byte[] payload, boolean presenceFieldsTest) throws Exception {
client = getCoapClient(FeatureType.ATTRIBUTES);
postAttributes(client, payload);
client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES);
CoapResponse coapResponse = client.postMethod(payload);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
DeviceId deviceId = savedDevice.getId();
long start = System.currentTimeMillis();
long end = System.currentTimeMillis() + 5000;
List<String> actualKeys = null;
while (start <= end) {
actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {});
if (actualKeys.size() == expectedKeys.size()) {
break;
}
Thread.sleep(100);
start += 100;
}
List<String> actualKeys = getActualKeysList(deviceId, expectedKeys);
assertNotNull(actualKeys);
Set<String> actualKeySet = new HashSet<>(actualKeys);
Set<String> expectedKeySet = new HashSet<>(expectedKeys);
assertEquals(expectedKeySet, actualKeySet);
String getAttributesValuesUrl = getAttributesValuesUrl(deviceId, actualKeySet);
@ -111,14 +95,6 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest {
doDelete(deleteAttributesUrl);
}
private void postAttributes(CoapClient client, byte[] payload) throws IOException, ConnectorException {
if (payload == null) {
payload = PAYLOAD_VALUES_STR.getBytes();
}
CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
}
@SuppressWarnings({"unchecked", "rawtypes"})
protected void assertAttributesValues(List<Map<String, Object>> deviceValues, Set<String> keySet) {
for (Map<String, Object> map : deviceValues) {
@ -171,6 +147,22 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest {
}
}
private List<String> getActualKeysList(DeviceId deviceId, List<String> expectedKeys) throws Exception {
long start = System.currentTimeMillis();
long end = System.currentTimeMillis() + 5000;
List<String> actualKeys = null;
while (start <= end) {
actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {});
if (actualKeys.size() == expectedKeys.size()) {
break;
}
Thread.sleep(100);
start += 100;
}
return actualKeys;
}
private String getAttributesValuesUrl(DeviceId deviceId, Set<String> actualKeySet) {
return "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/attributes/CLIENT_SCOPE?keys=" + String.join(",", actualKeySet);
}

53
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java

@ -17,19 +17,17 @@ package org.thingsboard.server.transport.coap.telemetry.timeseries;
import com.fasterxml.jackson.core.type.TypeReference;
import lombok.extern.slf4j.Slf4j;
import org.eclipse.californium.core.CoapClient;
import org.eclipse.californium.core.CoapResponse;
import org.eclipse.californium.core.coap.CoAP;
import org.eclipse.californium.core.coap.MediaTypeRegistry;
import org.eclipse.californium.elements.exception.ConnectorException;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.id.DeviceId;
import org.thingsboard.server.common.msg.session.FeatureType;
import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest;
import org.thingsboard.server.transport.coap.CoapTestClient;
import org.thingsboard.server.transport.coap.CoapTestConfigProperties;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
@ -75,28 +73,16 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap
}
protected void processTestPostTelemetry(byte[] payloadBytes, List<String> expectedKeys, boolean withTs, boolean presenceFieldsTest) throws Exception {
client = getCoapClient(FeatureType.TELEMETRY);
postTelemetry(client, payloadBytes);
String deviceId = savedDevice.getId().getId().toString();
long start = System.currentTimeMillis();
long end = System.currentTimeMillis() + 5000;
client = new CoapTestClient(accessToken, FeatureType.TELEMETRY);
CoapResponse coapResponse = client.postMethod(payloadBytes);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
List<String> actualKeys = null;
while (start <= end) {
actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {});
if (actualKeys.size() == expectedKeys.size()) {
break;
}
Thread.sleep(100);
start += 100;
}
DeviceId deviceId = savedDevice.getId();
List<String> actualKeys = getActualKeysList(deviceId, expectedKeys);
assertNotNull(actualKeys);
Set<String> actualKeySet = new HashSet<>(actualKeys);
Set<String> expectedKeySet = new HashSet<>(expectedKeys);
assertEquals(expectedKeySet, actualKeySet);
String getTelemetryValuesUrl;
@ -105,8 +91,8 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap
} else {
getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet);
}
start = System.currentTimeMillis();
end = System.currentTimeMillis() + 5000;
long start = System.currentTimeMillis();
long end = System.currentTimeMillis() + 5000;
Map<String, List<Map<String, Object>>> values = null;
while (start <= end) {
values = doGetAsyncTyped(getTelemetryValuesUrl, new TypeReference<>() {});
@ -144,11 +130,6 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap
}
}
private void postTelemetry(CoapClient client, byte[] payload) throws IOException, ConnectorException {
CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON);
assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode());
}
private void assertTs(Map<String, List<Map<String, Object>>> deviceValues, List<String> expectedKeys, int ts, int arrayIndex) {
assertEquals(ts, deviceValues.get(expectedKeys.get(0)).get(arrayIndex).get("ts"));
assertEquals(ts, deviceValues.get(expectedKeys.get(1)).get(arrayIndex).get("ts"));
@ -207,4 +188,20 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap
}
}
private List<String> getActualKeysList(DeviceId deviceId, List<String> expectedKeys) throws Exception {
long start = System.currentTimeMillis();
long end = System.currentTimeMillis() + 5000;
List<String> actualKeys = null;
while (start <= end) {
actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {});
if (actualKeys.size() == expectedKeys.size()) {
break;
}
Thread.sleep(100);
start += 100;
}
return actualKeys;
}
}

1
application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java

@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.CoapDeviceType;
import org.thingsboard.server.common.data.DeviceProfileProvisionType;
import org.thingsboard.server.common.data.TransportPayloadType;
import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration;
import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration;

2
common/cache/src/main/java/org/thingsboard/server/cache/SimpleTbCacheValueWrapper.java

@ -17,8 +17,10 @@ package org.thingsboard.server.cache;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.ToString;
import org.springframework.cache.Cache;
@ToString
@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public class SimpleTbCacheValueWrapper<T> implements TbCacheValueWrapper<T> {

43
common/cluster-api/src/main/proto/queue.proto

@ -454,6 +454,14 @@ message GetOtaPackageResponseMsg {
string fileName = 8;
}
message DeviceActivityProto {
int64 tenantIdMSB = 1;
int64 tenantIdLSB = 2;
int64 deviceIdMSB = 3;
int64 deviceIdLSB = 4;
int64 lastActivityTime = 5;
}
//Used to report session state to tb-Service and persist this state in the cache on the tb-Service level.
message SubscriptionInfoProto {
int64 lastActivityTime = 1;
@ -707,7 +715,10 @@ message PrepareMsg {
message AddMsg {
string relativePath = 1;
string entityDataJson = 2;
string entityDataJsonChunk = 2;
string chunkedMsgId = 3;
int32 chunkIndex = 4;
int32 chunksCount = 5;
}
message DeleteMsg {
@ -747,9 +758,8 @@ message ListVersionsResponseMsg {
}
message ListEntitiesRequestMsg {
string branchName = 1;
string versionId = 2;
string entityType = 3;
string versionId = 1;
string entityType = 2;
}
message VersionedEntityInfoProto {
@ -765,8 +775,13 @@ message ListEntitiesResponseMsg {
message ListBranchesRequestMsg {
}
message BranchInfoProto {
string name = 1;
bool isDefault = 2;
}
message ListBranchesResponseMsg {
repeated string branches = 1;
repeated BranchInfoProto branches = 1;
}
message EntityContentRequestMsg {
@ -778,6 +793,9 @@ message EntityContentRequestMsg {
message EntityContentResponseMsg {
string data = 1;
string chunkedMsgId = 2;
int32 chunkIndex = 3;
int32 chunksCount = 4;
}
message EntitiesContentRequestMsg {
@ -788,7 +806,8 @@ message EntitiesContentRequestMsg {
}
message EntitiesContentResponseMsg {
repeated string data = 1;
EntityContentResponseMsg item = 1;
int32 itemsCount = 2;
}
message VersionsDiffRequestMsg {
@ -810,15 +829,6 @@ message EntityVersionsDiff {
string rawDiff = 6;
}
message ContentsDiffRequestMsg {
string content1 = 1;
string content2 = 2;
}
message ContentsDiffResponseMsg {
string diff = 1;
}
message GenericRepositoryRequestMsg {}
message GenericRepositoryResponseMsg {}
@ -840,7 +850,6 @@ message ToVersionControlServiceMsg {
EntityContentRequestMsg entityContentRequest = 14;
EntitiesContentRequestMsg entitiesContentRequest = 15;
VersionsDiffRequestMsg versionsDiffRequest = 16;
ContentsDiffRequestMsg contentsDiffRequest = 17;
}
message VersionControlResponseMsg {
@ -855,7 +864,6 @@ message VersionControlResponseMsg {
EntityContentResponseMsg entityContentResponse = 9;
EntitiesContentResponseMsg entitiesContentResponse = 10;
VersionsDiffResponseMsg versionsDiffResponse = 11;
ContentsDiffResponseMsg contentsDiffResponse = 12;
}
/**
@ -902,6 +910,7 @@ message ToCoreMsg {
SubscriptionMgrMsgProto toSubscriptionMgrMsg = 3;
bytes toDeviceActorNotificationMsg = 4;
EdgeNotificationMsgProto edgeNotificationMsg = 5;
DeviceActivityProto deviceActivityMsg = 6;
}
/* High priority messages with low latency are handled by ThingsBoard Core Service separately */

1
common/dao-api/src/main/java/org/thingsboard/server/dao/entityview/EntityViewService.java

@ -27,7 +27,6 @@ import org.thingsboard.server.common.data.id.EntityViewId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.TimePageLink;
import java.util.List;

6
common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java

@ -15,6 +15,8 @@
*/
package org.thingsboard.server.common.data;
import com.google.common.base.Splitter;
import static org.apache.commons.lang3.StringUtils.repeat;
public class StringUtils {
@ -92,4 +94,8 @@ public class StringUtils {
return input.substring(0, startIndexInclusive) + obfuscatedPart + input.substring(endIndexExclusive);
}
public static Iterable<String> split(String value, int maxPartSize) {
return Splitter.fixedLength(maxPartSize).split(value);
}
}

39
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/BranchInfo.java

@ -0,0 +1,39 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.data.sync.vc;
import lombok.Data;
import java.util.Objects;
@Data
public class BranchInfo {
private final String name;
private final boolean isDefault;
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
BranchInfo that = (BranchInfo) o;
return Objects.equals(name, that.name);
}
@Override
public int hashCode() {
return Objects.hash(name);
}
}

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java

@ -24,5 +24,4 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData;
public class EntityDataDiff {
private EntityExportData<?> currentVersion;
private EntityExportData<?> otherVersion;
private String rawDiff;
}

5
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java

@ -20,12 +20,15 @@ import lombok.Builder;
import lombok.Data;
import org.thingsboard.server.common.data.id.EntityId;
import java.io.Serializable;
import java.util.List;
@Data
@Builder
@JsonInclude(JsonInclude.Include.NON_NULL)
public class EntityLoadError {
public class EntityLoadError implements Serializable {
private static final long serialVersionUID = 7538450180582109391L;
private String type;
private EntityId source;

6
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java

@ -21,11 +21,15 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import org.thingsboard.server.common.data.EntityType;
import java.io.Serializable;
@Data
@AllArgsConstructor
@NoArgsConstructor
@Builder
public class EntityTypeLoadResult {
public class EntityTypeLoadResult implements Serializable {
private static final long serialVersionUID = -8428039809651395241L;
private EntityType entityType;
private int created;
private int updated;

1
common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java

@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.EntityId;
@NoArgsConstructor
public class VersionedEntityInfo {
private EntityId externalId;
// etc..
public VersionedEntityInfo(EntityId externalId) {
this.externalId = externalId;

20
common/message/src/main/java/org/thingsboard/server/common/msg/plugin/ComponentLifecycleListener.java

@ -0,0 +1,20 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.common.msg.plugin;
public interface ComponentLifecycleListener {
void onComponentLifecycleMsg(ComponentLifecycleMsg componentLifecycleMsg);
}

7
common/message/src/test/java/org/thingsboard/server/common/msg/tools/RateLimitsTest.java

@ -15,8 +15,11 @@
*/
package org.thingsboard.server.common.msg.tools;
import org.awaitility.pollinterval.FixedPollInterval;
import org.awaitility.pollinterval.PollInterval;
import org.junit.Test;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
import static org.assertj.core.api.Assertions.assertThat;
@ -39,10 +42,11 @@ public class RateLimitsTest {
assertThat(rateLimits.tryConsume()).as("new token is available").isFalse();
int expectedRefillTime = (int) (((double) period / capacity) * 1000);
int gap = 100;
int gap = 500;
for (int i = 0; i < capacity; i++) {
await("token refill for rate limit " + rateLimitConfig)
.pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS))
.atLeast(expectedRefillTime - gap, TimeUnit.MILLISECONDS)
.atMost(expectedRefillTime + gap, TimeUnit.MILLISECONDS)
.untilAsserted(() -> {
@ -70,6 +74,7 @@ public class RateLimitsTest {
int gap = 500;
await("tokens refill for rate limit " + rateLimitConfig)
.pollInterval(new FixedPollInterval(10, TimeUnit.MILLISECONDS))
.atLeast(expectedRefillTime - gap, TimeUnit.MILLISECONDS)
.atMost(expectedRefillTime + gap, TimeUnit.MILLISECONDS)
.untilAsserted(() -> {

8
common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java

@ -39,4 +39,12 @@ public class CollectionsUtil {
return isNotEmpty(collection) && collection.contains(element);
}
public static <T> int countNonNull(T[] array) {
int count = 0;
for (T t : array) {
if (t != null) count++;
}
return count;
}
}

74
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java

@ -15,6 +15,7 @@
*/
package org.thingsboard.server.service.sync.vc;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@ -28,6 +29,7 @@ import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Service;
import org.thingsboard.common.util.CollectionsUtil;
import org.thingsboard.common.util.ThingsBoardThreadFactory;
import org.thingsboard.server.common.data.EntityType;
import org.thingsboard.server.common.data.StringUtils;
@ -42,6 +44,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType;
import org.thingsboard.server.common.msg.queue.TopicPartitionInfo;
import org.thingsboard.server.gen.transport.TransportProtos;
import org.thingsboard.server.gen.transport.TransportProtos.AddMsg;
import org.thingsboard.server.gen.transport.TransportProtos.BranchInfoProto;
import org.thingsboard.server.gen.transport.TransportProtos.CommitRequestMsg;
import org.thingsboard.server.gen.transport.TransportProtos.CommitResponseMsg;
import org.thingsboard.server.gen.transport.TransportProtos.DeleteMsg;
@ -88,6 +91,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
@ -122,6 +126,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
private long packProcessingTimeout;
@Value("${vc.git.io_pool_size:3}")
private int ioPoolSize;
@Value("${queue.vc.msg-chunk-size:500000}")
private int msgChunkSize;
//We need to manually manage the threads since tasks for particular tenant need to be processed sequentially.
private final List<ListeningExecutorService> ioThreads = new ArrayList<>();
@ -251,8 +257,6 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest());
} else if (msg.hasVersionsDiffRequest()) {
handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest());
} else if (msg.hasContentsDiffRequest()) {
handleContentsDiffRequest(ctx, msg.getContentsDiffRequest());
}
}
}
@ -269,19 +273,49 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
String path = getRelativePath(entityType, null);
var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path)
.stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList());
var response = EntitiesContentResponseMsg.newBuilder();
for (VersionedEntityInfo info : ids) {
var data = vcService.getFileContentAtCommit(ctx.getTenantId(),
getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId());
response.addData(data);
if (!ids.isEmpty()) {
for (VersionedEntityInfo info : ids) {
var data = vcService.getFileContentAtCommit(ctx.getTenantId(),
getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId());
Iterable<String> dataChunks = StringUtils.split(data, msgChunkSize);
String chunkedMsgId = UUID.randomUUID().toString();
int chunksCount = Iterables.size(dataChunks);
AtomicInteger chunkIndex = new AtomicInteger();
dataChunks.forEach(chunk -> {
EntitiesContentResponseMsg.Builder response = EntitiesContentResponseMsg.newBuilder()
.setItemsCount(ids.size())
.setItem(EntityContentResponseMsg.newBuilder()
.setData(chunk)
.setChunkedMsgId(chunkedMsgId)
.setChunksCount(chunksCount)
.setChunkIndex(chunkIndex.getAndIncrement())
.build());
reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response));
});
}
} else {
reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(
EntitiesContentResponseMsg.newBuilder()
.setItemsCount(0)));
}
reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response));
}
private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException {
String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString());
String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId());
reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data)));
Iterable<String> dataChunks = StringUtils.split(data, msgChunkSize);
String chunkedMsgId = UUID.randomUUID().toString();
int chunksCount = Iterables.size(dataChunks);
AtomicInteger chunkIndex = new AtomicInteger();
dataChunks.forEach(chunk -> {
log.trace("[{}] sending chunk {} for 'getEntity'", chunkedMsgId, chunkIndex.get());
reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder()
.setData(chunk).setChunkedMsgId(chunkedMsgId).setChunksCount(chunksCount)
.setChunkIndex(chunkIndex.getAndIncrement())));
});
}
private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception {
@ -332,7 +366,10 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
private void handleListBranches(VersionControlRequestCtx ctx, ListBranchesRequestMsg request) {
var branches = vcService.listBranches(ctx.getTenantId());
var branches = vcService.listBranches(ctx.getTenantId()).stream()
.map(branchInfo -> BranchInfoProto.newBuilder()
.setName(branchInfo.getName())
.setIsDefault(branchInfo.isDefault()).build()).collect(Collectors.toList());
reply(ctx, Optional.empty(), builder -> builder.setListBranchesResponse(ListBranchesResponseMsg.newBuilder().addAllBranches(branches)));
}
@ -355,12 +392,6 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
.addAllDiff(diffList)));
}
private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException {
String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2());
reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder()
.setDiff(diff)));
}
private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception {
var tenantId = ctx.getTenantId();
UUID txId = UUID.fromString(request.getTxId());
@ -412,7 +443,16 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe
}
private void addToCommit(VersionControlRequestCtx ctx, PendingCommit commit, AddMsg addMsg) throws IOException {
vcService.add(commit, addMsg.getRelativePath(), addMsg.getEntityDataJson());
log.trace("[{}] received chunk {} for 'addToCommit'", addMsg.getChunkedMsgId(), addMsg.getChunkIndex());
Map<String, String[]> chunkedMsgs = commit.getChunkedMsgs();
String[] msgChunks = chunkedMsgs.computeIfAbsent(addMsg.getChunkedMsgId(), id -> new String[addMsg.getChunksCount()]);
msgChunks[addMsg.getChunkIndex()] = addMsg.getEntityDataJsonChunk();
if (CollectionsUtil.countNonNull(msgChunks) == msgChunks.length) {
log.trace("[{}] collected all chunks for 'addToCommit'", addMsg.getChunkedMsgId());
String entityDataJson = String.join("", msgChunks);
chunkedMsgs.remove(addMsg.getChunkedMsgId());
vcService.add(commit, addMsg.getRelativePath(), entityDataJson);
}
}
private void doAbortCurrentCommit(TenantId tenantId, PendingCommit current) {

7
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitRepositoryService.java

@ -29,8 +29,9 @@ import org.thingsboard.server.common.data.id.EntityIdFactory;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.GitRepository.Diff;
@ -84,7 +85,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
repository.createAndCheckoutOrphanBranch(commit.getWorkingBranch());
repository.resetAndClean();
if (repository.listRemoteBranches().contains(branch)) {
if (repository.listRemoteBranches().contains(new BranchInfo(branch, false))) {
repository.merge(branch);
}
} catch (IOException | GitAPIException gitAPIException) {
@ -182,7 +183,7 @@ public class DefaultGitRepositoryService implements GitRepositoryService {
}
@Override
public List<String> listBranches(TenantId tenantId) {
public List<BranchInfo> listBranches(TenantId tenantId) {
GitRepository repository = checkRepository(tenantId);
try {
return repository.listRemoteBranches();

31
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java

@ -41,10 +41,12 @@ import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectLoader;
import org.eclipse.jgit.lib.ObjectReader;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.revwalk.filter.RevFilter;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.FetchResult;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.SshTransport;
import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider;
@ -58,6 +60,7 @@ import org.eclipse.jgit.treewalk.filter.PathFilter;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.page.SortOrder;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.RepositoryAuthMethod;
@ -69,7 +72,12 @@ import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.security.KeyPair;
import java.security.PublicKey;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -84,6 +92,8 @@ public class GitRepository {
@Getter
private final String directory;
private ObjectId headId;
private GitRepository(Git git, RepositorySettings settings, CredentialsProvider credentialsProvider, SshdSessionFactory sshSessionFactory, String directory) {
this.git = git;
this.settings = settings;
@ -135,8 +145,12 @@ public class GitRepository {
}
public void fetch() throws GitAPIException {
execute(git.fetch()
FetchResult result = execute(git.fetch()
.setRemoveDeletedRefs(true));
Ref head = result.getAdvertisedRef(Constants.HEAD);
if (head != null) {
this.headId = head.getObjectId();
}
}
public void deleteLocalBranchIfExists(String branch) throws GitAPIException {
@ -162,13 +176,11 @@ public class GitRepository {
.include(branchId));
}
public List<String> listRemoteBranches() throws GitAPIException {
public List<BranchInfo> listRemoteBranches() throws GitAPIException {
return execute(git.branchList()
.setListMode(ListBranchCommand.ListMode.REMOTE)).stream()
.filter(ref -> !ref.getName().equals(Constants.HEAD))
.map(ref -> org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName()))
.map(name -> StringUtils.removeStart(name, "origin/"))
.map(this::toBranchInfo)
.distinct().collect(Collectors.toList());
}
@ -325,6 +337,13 @@ public class GitRepository {
.collect(Collectors.toList());
}
private BranchInfo toBranchInfo(Ref ref) {
String name = org.eclipse.jgit.lib.Repository.shortenRefName(ref.getName());
String branchName = StringUtils.removeStart(name, "origin/");
boolean isDefault = this.headId != null && this.headId.equals(ref.getObjectId());
return new BranchInfo(branchName, isDefault);
}
private Commit toCommit(RevCommit revCommit) {
return new Commit(revCommit.getCommitTime() * 1000l, revCommit.getName(),
revCommit.getFullMessage(), revCommit.getAuthorIdent().getName(), revCommit.getAuthorIdent().getEmailAddress());

5
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepositoryService.java

@ -19,8 +19,9 @@ import org.eclipse.jgit.api.errors.GitAPIException;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.page.PageData;
import org.thingsboard.server.common.data.page.PageLink;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.BranchInfo;
import org.thingsboard.server.common.data.sync.vc.EntityVersion;
import org.thingsboard.server.common.data.sync.vc.RepositorySettings;
import org.thingsboard.server.common.data.sync.vc.VersionCreationResult;
import org.thingsboard.server.common.data.sync.vc.VersionedEntityInfo;
import org.thingsboard.server.service.sync.vc.GitRepository.Diff;
@ -57,7 +58,7 @@ public interface GitRepositoryService {
void abort(PendingCommit commit);
List<String> listBranches(TenantId tenantId);
List<BranchInfo> listBranches(TenantId tenantId);
String getFileContentAtCommit(TenantId tenantId, String relativePath, String versionId) throws IOException;

13
common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java

@ -18,7 +18,9 @@ package org.thingsboard.server.service.sync.vc;
import lombok.Data;
import org.thingsboard.server.common.data.id.TenantId;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
@Data
public class PendingCommit {
@ -31,9 +33,10 @@ public class PendingCommit {
private String versionName;
private String authorName;
private String authorEmail;
private Map<String, String[]> chunkedMsgs;
public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) {
this.tenantId = tenantId;
this.nodeId = nodeId;
@ -44,4 +47,12 @@ public class PendingCommit {
this.authorEmail = authorEmail;
this.workingBranch = txId.toString();
}
public Map<String, String[]> getChunkedMsgs() {
if (chunkedMsgs == null) {
chunkedMsgs = new ConcurrentHashMap<>();
}
return chunkedMsgs;
}
}

2
dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileServiceImpl.java

@ -18,6 +18,7 @@ package org.thingsboard.server.dao.device;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.springframework.transaction.event.TransactionalEventListener;
import org.thingsboard.server.common.data.Device;
@ -69,6 +70,7 @@ public class DeviceProfileServiceImpl extends AbstractCachedEntityService<Device
@Autowired
private DataValidator<DeviceProfile> deviceProfileValidator;
@Lazy
@Autowired
private QueueService queueService;

2
dao/src/main/java/org/thingsboard/server/dao/queue/BaseQueueService.java

@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Service;
import org.thingsboard.server.common.data.TenantProfile;
import org.thingsboard.server.common.data.id.QueueId;
@ -43,6 +44,7 @@ public class BaseQueueService extends AbstractEntityService implements QueueServ
@Autowired
private QueueDao queueDao;
@Lazy
@Autowired
private TbTenantProfileCache tenantProfileCache;

2
dao/src/main/java/org/thingsboard/server/dao/service/validator/DeviceProfileDataValidator.java

@ -90,8 +90,8 @@ public class DeviceProfileDataValidator extends DataValidator<DeviceProfile> {
private DeviceDao deviceDao;
@Autowired
private TenantService tenantService;
@Autowired
@Lazy
@Autowired
private QueueService queueService;
@Autowired
private OtaPackageService otaPackageService;

18
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java

@ -26,6 +26,7 @@ import org.springframework.data.domain.Sort;
import org.thingsboard.server.common.data.id.EntityId;
import org.thingsboard.server.common.data.id.TenantId;
import org.thingsboard.server.common.data.kv.Aggregation;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.DeleteTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
@ -122,14 +123,17 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(entityId, query);
} else {
long stepTs = query.getStartTs();
List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
while (stepTs < query.getEndTs()) {
long startTs = stepTs;
long endTs = stepTs + query.getInterval();
long endPeriod = query.getEndTs();
long startPeriod = query.getStartTs();
long step = query.getInterval();
while (startPeriod <= endPeriod) {
long startTs = startPeriod;
long endTs = Math.min(startPeriod + step, endPeriod + 1);
long ts = startTs + (endTs - startTs) / 2;
futures.add(findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation()));
stepTs = endTs;
ListenableFuture<Optional<TsKvEntry>> aggregateTsKvEntry = findAndAggregateAsync(entityId, query.getKey(), startTs, endTs, ts, query.getAggregation());
futures.add(aggregateTsKvEntry);
startPeriod = endTs;
}
return getTskvEntriesFuture(Futures.allAsList(futures));
}
@ -148,7 +152,7 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq
return Futures.immediateFuture(DaoUtil.convertDataList(tsKvEntities));
}
private ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
ListenableFuture<Optional<TsKvEntry>> findAndAggregateAsync(EntityId entityId, String key, long startTs, long endTs, long ts, Aggregation aggregation) {
List<CompletableFuture<TsKvEntity>> entitiesFutures = new ArrayList<>();
switchAggregation(entityId, key, startTs, endTs, aggregation, entitiesFutures);
return Futures.transform(setFutures(entitiesFutures), entity -> {

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractSqlTimeseriesDao.java

@ -91,7 +91,7 @@ public abstract class AbstractSqlTimeseriesDao extends BaseAbstractSqlTimeseries
.stream()
.map(query -> findAllAsync(tenantId, entityId, query))
.collect(Collectors.toList());
return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() {
return Futures.transform(Futures.allAsList(futures), new Function<>() {
@Nullable
@Override
public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) {

2
dao/src/main/java/org/thingsboard/server/dao/sqlts/AggregationTimeseriesDao.java

@ -26,4 +26,4 @@ import java.util.List;
public interface AggregationTimeseriesDao {
ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query);
}
}

6
dao/src/main/java/org/thingsboard/server/dao/sqlts/BaseAbstractSqlTimeseriesDao.java

@ -21,6 +21,7 @@ import com.google.common.util.concurrent.ListenableFuture;
import lombok.extern.slf4j.Slf4j;
import org.hibernate.exception.ConstraintViolationException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.dao.DataIntegrityViolationException;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionary;
import org.thingsboard.server.dao.model.sqlts.dictionary.TsKvDictionaryCompositeKey;
@ -39,9 +40,7 @@ import java.util.stream.Collectors;
public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeningExecutorService {
private final ConcurrentMap<String, Integer> tsKvDictionaryMap = new ConcurrentHashMap<>();
protected static final ReentrantLock tsCreationLock = new ReentrantLock();
@Autowired
protected TsKvDictionaryRepository dictionaryRepository;
@ -61,7 +60,7 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni
TsKvDictionary saved = dictionaryRepository.save(tsKvDictionary);
tsKvDictionaryMap.put(saved.getKey(), saved.getKeyId());
keyId = saved.getKeyId();
} catch (ConstraintViolationException e) {
} catch (DataIntegrityViolationException | ConstraintViolationException e) {
tsKvDictionaryOptional = dictionaryRepository.findById(new TsKvDictionaryCompositeKey(strKey));
TsKvDictionary dictionary = tsKvDictionaryOptional.orElseThrow(() -> new RuntimeException("Failed to get TsKvDictionary entity from DB!"));
tsKvDictionaryMap.put(dictionary.getKey(), dictionary.getKeyId());
@ -96,4 +95,5 @@ public abstract class BaseAbstractSqlTimeseriesDao extends JpaAbstractDaoListeni
}
}, service);
}
}

1
dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java

@ -107,6 +107,7 @@ public class TenantServiceImpl extends AbstractCachedEntityService<TenantCacheKe
@Autowired
private DataValidator<Tenant> tenantValidator;
@Lazy
@Autowired
private QueueService queueService;

18
dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java

@ -141,7 +141,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
@Override
public ListenableFuture<List<TsKvEntry>> findAllAsync(TenantId tenantId, EntityId entityId, List<ReadTsKvQuery> queries) {
List<ListenableFuture<List<TsKvEntry>>> futures = queries.stream().map(query -> findAllAsync(tenantId, entityId, query)).collect(Collectors.toList());
return Futures.transform(Futures.allAsList(futures), new Function<List<List<TsKvEntry>>, List<TsKvEntry>>() {
return Futures.transform(Futures.allAsList(futures), new Function<>() {
@Nullable
@Override
public List<TsKvEntry> apply(@Nullable List<List<TsKvEntry>> results) {
@ -270,18 +270,20 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD
if (query.getAggregation() == Aggregation.NONE) {
return findAllAsyncWithLimit(tenantId, entityId, query);
} else {
long startPeriod = query.getStartTs();
long endPeriod = query.getEndTs();
long step = Math.max(query.getInterval(), MIN_AGGREGATION_STEP_MS);
long stepTs = query.getStartTs();
List<ListenableFuture<Optional<TsKvEntry>>> futures = new ArrayList<>();
while (stepTs < query.getEndTs()) {
long startTs = stepTs;
long endTs = stepTs + step;
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, step, 1, query.getAggregation(), query.getOrder());
while (startPeriod <= endPeriod) {
long startTs = startPeriod;
long endTs = Math.min(startPeriod + step, endPeriod + 1);
long ts = endTs - startTs;
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(query.getKey(), startTs, endTs, ts, 1, query.getAggregation(), query.getOrder());
futures.add(findAndAggregateAsync(tenantId, entityId, subQuery, toPartitionTs(startTs), toPartitionTs(endTs)));
stepTs = endTs;
startPeriod = endTs;
}
ListenableFuture<List<Optional<TsKvEntry>>> future = Futures.allAsList(futures);
return Futures.transform(future, new Function<List<Optional<TsKvEntry>>, List<TsKvEntry>>() {
return Futures.transform(future, new Function<>() {
@Nullable
@Override
public List<TsKvEntry> apply(@Nullable List<Optional<TsKvEntry>> input) {

9
dao/src/main/java/org/thingsboard/server/dao/util/AbstractBufferedRateExecutor.java

@ -116,8 +116,13 @@ public abstract class AbstractBufferedRateExecutor<T extends AsyncTask, F extend
F result = wrap(task, settableFuture);
boolean perTenantLimitReached = false;
var tenantProfileConfiguration = tenantProfileCache.get(task.getTenantId()).getDefaultProfileConfiguration();
if (StringUtils.isNotEmpty(tenantProfileConfiguration.getCassandraQueryTenantRateLimitsConfiguration())) {
var tenantProfileConfiguration =
(task.getTenantId() != null && !TenantId.SYS_TENANT_ID.equals(task.getTenantId()))
? tenantProfileCache.get(task.getTenantId()).getDefaultProfileConfiguration()
: null;
if (tenantProfileConfiguration != null &&
StringUtils.isNotEmpty(tenantProfileConfiguration.getCassandraQueryTenantRateLimitsConfiguration())) {
if (task.getTenantId() == null) {
log.info("Invalid task received: {}", task);
} else if (!task.getTenantId().isNullUid()) {

134
dao/src/test/java/org/thingsboard/server/dao/service/timeseries/BaseTimeseriesServiceTest.java

@ -184,6 +184,140 @@ public abstract class BaseTimeseriesServiceTest extends AbstractServiceTest {
Assert.assertEquals(toTsEntry(TS - 1, stringKvEntry), entries.get(2));
}
@Test
public void testFindByQuery_whenPeriodEqualsOneMilisecondPeriod() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
saveEntries(deviceId, TS);
saveEntries(deviceId, TS + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS, 1, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS, new LongDataEntry(LONG_KEY, 1L)), entries.get(0));
}
@Test
public void testFindByQuery_whenPeriodEqualsInterval() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 100L; i += 10L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 100L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 100, 101, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(1, entries.size());
Assert.assertEquals(toTsEntry(TS + 50, new LongDataEntry(LONG_KEY, 11L)), entries.get(0));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 100000L; i += 10000L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 100000L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
saveEntries(deviceId, TS - 1L);
for (long i = TS; i <= TS + 80000L; i += 10000L) {
saveEntries(deviceId, i);
}
saveEntries(deviceId, TS + 80000L + 1L);
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 4L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoIntervalWithEqualsLength_whereNotAllEntriesInRange() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000) {
saveEntries(deviceId, i);
}
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 99999, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 75000, new LongDataEntry(LONG_KEY, 5L)), entries.get(1));
}
@Test
public void testFindByQuery_whenPeriodHaveTwoInterval_whereSecondShorterThanFirst_andNotAllEntriesInRange() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());
for (long i = TS - 1L; i <= TS + 100000L + 1L; i += 10000L) {
saveEntries(deviceId, i);
}
List<ReadTsKvQuery> queries = List.of(new BaseReadTsKvQuery(LONG_KEY, TS, TS + 80000, 50000, 1, Aggregation.COUNT, DESC_ORDER));
List<TsKvEntry> entries = tsService.findAll(tenantId, deviceId, queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1));
EntityView entityView = saveAndCreateEntityView(deviceId, List.of(LONG_KEY));
entries = tsService.findAll(tenantId, entityView.getId(), queries).get();
Assert.assertEquals(2, entries.size());
Assert.assertEquals(toTsEntry(TS + 25000, new LongDataEntry(LONG_KEY, 5L)), entries.get(0));
Assert.assertEquals(toTsEntry(TS + 65000, new LongDataEntry(LONG_KEY, 3L)), entries.get(1));
}
@Test
public void testFindByQueryDescOrder() throws Exception {
DeviceId deviceId = new DeviceId(Uuids.timeBased());

156
dao/src/test/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDaoTest.java

@ -0,0 +1,156 @@
/**
* Copyright © 2016-2022 The Thingsboard Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.thingsboard.server.dao.sqlts;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import org.junit.Before;
import org.junit.Test;
import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery;
import org.thingsboard.server.common.data.kv.ReadTsKvQuery;
import org.thingsboard.server.common.data.kv.TsKvEntry;
import java.util.Optional;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.BDDMockito.willCallRealMethod;
import static org.mockito.BDDMockito.willReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.thingsboard.server.common.data.id.TenantId.SYS_TENANT_ID;
import static org.thingsboard.server.common.data.kv.Aggregation.COUNT;
public class AbstractChunkedAggregationTimeseriesDaoTest {
final int LIMIT = 1;
final String TEMP = "temp";
final String DESC = "DESC";
AbstractChunkedAggregationTimeseriesDao tsDao;
@Before
public void setUp() throws Exception {
tsDao = spy(AbstractChunkedAggregationTimeseriesDao.class);
ListenableFuture<Optional<TsKvEntry>> optionalListenableFuture = Futures.immediateFuture(Optional.of(mock(TsKvEntry.class)));
willReturn(optionalListenableFuture).given(tsDao).findAndAggregateAsync(any(), anyString(), anyLong(), anyLong(), anyLong(), any());
willReturn(Futures.immediateFuture(mock(TsKvEntry.class))).given(tsDao).getTskvEntriesFuture(any());
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenLastIntervalShorterThanOthersAndEqualsEndTs() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2000, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 2001, 1001, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 2001, 3001, 2501, LIMIT, COUNT, DESC);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 2001, getTsForReadTsKvQuery(1, 2001), COUNT);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 2001, 3000 + 1, getTsForReadTsKvQuery(2001, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriod() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3000, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
assertThat(tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query)).isNotNull();
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000 + 1, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodMinusOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 2999, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3000, 1500, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuerySecond = new BaseReadTsKvQuery(TEMP, 3000, 3001, 3000, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(2)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3000, getTsForReadTsKvQuery(1, 3000), COUNT);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuerySecond.getKey(), 3000, 3001, getTsForReadTsKvQuery(3000, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsPeriodPlusOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3001, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsZero() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 0, 0, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 0, 1, 0, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 0, 1, getTsForReadTsKvQuery(0, 1), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsOne() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 1, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQuery = new BaseReadTsKvQuery(TEMP, 1, 2, 1, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQuery.getKey(), 1, 2, getTsForReadTsKvQuery(1, 2), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsOneMillisecondAndStartTsIsIntegerMax() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE, 1, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, Integer.MAX_VALUE, Integer.MAX_VALUE + 1L, Integer.MAX_VALUE, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), Integer.MAX_VALUE, 1L + Integer.MAX_VALUE, getTsForReadTsKvQuery(Integer.MAX_VALUE, 1L + Integer.MAX_VALUE), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenIntervalEqualsBigNumber() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, Integer.MAX_VALUE, LIMIT, COUNT, DESC);
ReadTsKvQuery subQueryFirst = new BaseReadTsKvQuery(TEMP, 1, 3001, 1501, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, subQueryFirst.getKey(), 1, 3001, getTsForReadTsKvQuery(1, 3001), COUNT);
}
@Test
public void givenIntervalNotMultiplePeriod_whenAggregateCount_thenCountIntervalEqualsPeriodSize() {
ReadTsKvQuery query = new BaseReadTsKvQuery(TEMP, 1, 3000, 3, LIMIT, COUNT, DESC);
willCallRealMethod().given(tsDao).findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
tsDao.findAllAsync(SYS_TENANT_ID, SYS_TENANT_ID, query);
verify(tsDao, times(1000)).findAndAggregateAsync(any(), any(), anyLong(), anyLong(), anyLong(), any());
for (long i = 1; i <= 3000; i += 3) {
ReadTsKvQuery querySub = new BaseReadTsKvQuery(TEMP, i, i + 3, i + (i + 3 - i) / 2, LIMIT, COUNT, DESC);
verify(tsDao, times(1)).findAndAggregateAsync(SYS_TENANT_ID, querySub.getKey(), i, i + 3, getTsForReadTsKvQuery(i, i + 3), COUNT);
}
}
long getTsForReadTsKvQuery(long startTs, long endTs) {
return startTs + (endTs - startTs) / 2L;
}
}

3
docker/.env

@ -1,5 +1,8 @@
TB_QUEUE_TYPE=kafka
# redis or redis-cluster
CACHE=redis
DOCKER_REPO=thingsboard
JS_EXECUTOR_DOCKER_NAME=tb-js-executor

8
docker/.gitignore

@ -6,4 +6,12 @@ tb-node/postgres/**
tb-node/cassandra/**
tb-transports/*/log
tb-vc-executor/log/**
tb-node/redis-cluster-data-0/**
tb-node/redis-cluster-data-1/**
tb-node/redis-cluster-data-2/**
tb-node/redis-cluster-data-3/**
tb-node/redis-cluster-data-4/**
tb-node/redis-cluster-data-5/**
tb-node/redis-data/**
!.env

Some files were not shown because too many files changed in this diff

Loading…
Cancel
Save