diff --git a/application/src/main/conf/logback.xml b/application/src/main/conf/logback.xml index 898128c1ff..6d2c95ee84 100644 --- a/application/src/main/conf/logback.xml +++ b/application/src/main/conf/logback.xml @@ -36,6 +36,8 @@ + + diff --git a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java index 6a616671eb..a425b0968a 100644 --- a/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java +++ b/application/src/main/java/org/thingsboard/server/controller/ControllerConstants.java @@ -138,6 +138,7 @@ public class ControllerConstants { protected static final String ENTITY_VERSION_TEXT_SEARCH_DESCRIPTION = "The case insensitive 'substring' filter based on the entity version name."; protected static final String VERSION_ID_PARAM_DESCRIPTION = "Version id, for example fd82625bdd7d6131cf8027b44ee967012ecaf990. Represents commit hash."; + protected static final String BRANCH_PARAM_DESCRIPTION = "The name of the working branch, for example 'master'"; protected static final String MARKDOWN_CODE_BLOCK_START = "```json\n"; protected static final String MARKDOWN_CODE_BLOCK_END = "\n```"; diff --git a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java index dfefce4b12..2e37bcbe59 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntitiesVersionControlController.java @@ -58,6 +58,9 @@ import java.util.List; import java.util.UUID; import java.util.stream.Collectors; +import static org.thingsboard.server.controller.ControllerConstants.BRANCH_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_ID_PARAM_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.ENTITY_TYPE_PARAM_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_END; import static org.thingsboard.server.controller.ControllerConstants.MARKDOWN_CODE_BLOCK_START; import static org.thingsboard.server.controller.ControllerConstants.NEW_LINE; @@ -230,8 +233,11 @@ public class EntitiesVersionControlController extends BaseController { MARKDOWN_CODE_BLOCK_END + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{entityType}/{externalEntityUuid}", params = {"branch", "pageSize", "page"}) - public DeferredResult> listEntityVersions(@PathVariable EntityType entityType, + public DeferredResult> listEntityVersions(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, + @ApiParam(value = "A string value representing external entity id. This is `externalId` property of an entity, or otherwise if not set - simply id of this entity.") @PathVariable UUID externalEntityUuid, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION) @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @@ -256,7 +262,9 @@ public class EntitiesVersionControlController extends BaseController { "The response structure is the same as for `listEntityVersions` API method." + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version/{entityType}", params = {"branch", "pageSize", "page"}) - public DeferredResult> listEntityTypeVersions(@PathVariable EntityType entityType, + public DeferredResult> listEntityTypeVersions(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @@ -279,7 +287,8 @@ public class EntitiesVersionControlController extends BaseController { "The response format is the same as for `listEntityVersions` API method." + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/version", params = {"branch", "pageSize", "page"}) - public DeferredResult> listVersions(@RequestParam String branch, + public DeferredResult> listVersions(@ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) + @RequestParam String branch, @ApiParam(value = PAGE_SIZE_DESCRIPTION, required = true) @RequestParam int pageSize, @ApiParam(value = PAGE_NUMBER_DESCRIPTION, required = true) @@ -302,9 +311,11 @@ public class EntitiesVersionControlController extends BaseController { "Entities order will be the same as in the repository." + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/entity/{entityType}/{versionId}", params = {"branch"}) - public DeferredResult> listEntitiesAtVersion(@PathVariable EntityType entityType, + public DeferredResult> listEntitiesAtVersion(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listEntitiesAtVersion(getTenantId(), branch, versionId, entityType)); @@ -318,6 +329,7 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping(value = "/entity/{versionId}", params = {"branch"}) public DeferredResult> listAllEntitiesAtVersion(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION, required = true) @RequestParam String branch) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); return wrapFuture(versionControlService.listAllEntitiesAtVersion(getTenantId(), branch, versionId)); @@ -332,7 +344,9 @@ public class EntitiesVersionControlController extends BaseController { @GetMapping("/info/{versionId}/{entityType}/{externalEntityUuid}") public DeferredResult getEntityDataInfo(@ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @PathVariable String versionId, + @ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) @PathVariable EntityType entityType, + @ApiParam(value = "A string value representing external entity id", required = true) @PathVariable UUID externalEntityUuid) throws Exception { accessControlService.checkPermission(getCurrentUser(), Resource.VERSION_CONTROL, Operation.READ); EntityId entityId = EntityIdFactory.getByTypeAndUuid(entityType, externalEntityUuid); @@ -344,8 +358,11 @@ public class EntitiesVersionControlController extends BaseController { "Entity data structure is the same as stored in a repository. " + TENANT_AUTHORITY_PARAGRAPH) @GetMapping(value = "/diff/{entityType}/{internalEntityUuid}", params = {"branch", "versionId"}) - public DeferredResult compareEntityDataToVersion(@PathVariable EntityType entityType, + public DeferredResult compareEntityDataToVersion(@ApiParam(value = ENTITY_TYPE_PARAM_DESCRIPTION, required = true) + @PathVariable EntityType entityType, + @ApiParam(value = ENTITY_ID_PARAM_DESCRIPTION, required = true) @PathVariable UUID internalEntityUuid, + @ApiParam(value = BRANCH_PARAM_DESCRIPTION) @RequestParam String branch, @ApiParam(value = VERSION_ID_PARAM_DESCRIPTION, required = true) @RequestParam String versionId) throws Exception { diff --git a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java index f279a29aac..7a95bf78f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java +++ b/application/src/main/java/org/thingsboard/server/service/component/AnnotationComponentDiscoveryService.java @@ -161,20 +161,20 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe configurationDescriptor.set("nodeDefinition", node); scannedComponent.setConfigurationDescriptor(configurationDescriptor); scannedComponent.setClazz(clazzName); - log.info("Processing scanned component: {}", scannedComponent); + log.debug("Processing scanned component: {}", scannedComponent); } catch (Exception e) { log.error("Can't initialize component {}, due to {}", def.getBeanClassName(), e.getMessage(), e); throw new RuntimeException(e); } ComponentDescriptor persistedComponent = componentDescriptorService.findByClazz(TenantId.SYS_TENANT_ID, clazzName); if (persistedComponent == null) { - log.info("Persisting new component: {}", scannedComponent); + log.debug("Persisting new component: {}", scannedComponent); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); } else if (scannedComponent.equals(persistedComponent)) { - log.info("Component is already persisted: {}", persistedComponent); + log.debug("Component is already persisted: {}", persistedComponent); scannedComponent = persistedComponent; } else { - log.info("Component {} will be updated to {}", persistedComponent, scannedComponent); + log.debug("Component {} will be updated to {}", persistedComponent, scannedComponent); componentDescriptorService.deleteByClazz(TenantId.SYS_TENANT_ID, persistedComponent.getClazz()); scannedComponent.setId(persistedComponent.getId()); scannedComponent = componentDescriptorService.saveComponent(TenantId.SYS_TENANT_ID, scannedComponent); @@ -224,7 +224,7 @@ public class AnnotationComponentDiscoveryService implements ComponentDiscoverySe @Override public void discoverComponents() { registerRuleNodeComponents(); - log.info("Found following definitions: {}", components.values()); + log.debug("Found following definitions: {}", components.values()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java index 7ee7696f1a..76477c974b 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/DefaultTbNotificationEntityService.java @@ -128,7 +128,7 @@ public class DefaultTbNotificationEntityService implements TbNotificationEntityS } @Override - public void notifyCreateOruUpdateTenant(Tenant tenant, ComponentLifecycleEvent event) { + public void notifyCreateOrUpdateTenant(Tenant tenant, ComponentLifecycleEvent event) { tbClusterService.onTenantChange(tenant, null); tbClusterService.broadcastEntityStateChangeEvent(tenant.getId(), tenant.getId(), event); } diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java index c1131761dc..5ce91046d6 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/TbNotificationEntityService.java @@ -71,7 +71,7 @@ public interface TbNotificationEntityService { E entity, ActionType actionType, SecurityUser user, Object... additionalInfo); - void notifyCreateOruUpdateTenant(Tenant tenant, ComponentLifecycleEvent event); + void notifyCreateOrUpdateTenant(Tenant tenant, ComponentLifecycleEvent event); void notifyDeleteTenant(Tenant tenant); diff --git a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java index 22baed8366..d3c12cb3f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java +++ b/application/src/main/java/org/thingsboard/server/service/entitiy/tenant/DefaultTbTenantService.java @@ -54,7 +54,7 @@ public class DefaultTbTenantService extends AbstractTbEntityService implements T installScripts.createDefaultEdgeRuleChains(savedTenant.getId()); } tenantProfileCache.evict(savedTenant.getId()); - notificationEntityService.notifyCreateOruUpdateTenant(savedTenant, created ? + notificationEntityService.notifyCreateOrUpdateTenant(savedTenant, created ? ComponentLifecycleEvent.CREATED : ComponentLifecycleEvent.UPDATED); TenantProfile oldTenantProfile = oldTenant != null ? tenantProfileService.findTenantProfileById(TenantId.SYS_TENANT_ID, oldTenant.getTenantProfileId()) : null; diff --git a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java index dc12326bdd..27931a9a64 100644 --- a/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/session/SessionRedisCache.java @@ -23,6 +23,7 @@ import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.cache.RedisTbTransactionalCache; @@ -33,14 +34,14 @@ import org.thingsboard.server.gen.transport.TransportProtos; public class SessionRedisCache extends RedisTbTransactionalCache { public SessionRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new RedisSerializer<>() { + super(CacheConstants.SESSIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>() { @Override public byte[] serialize(TransportProtos.DeviceSessionsCacheEntry deviceSessionsCacheEntry) throws SerializationException { return deviceSessionsCacheEntry.toByteArray(); } @Override - public TransportProtos.DeviceSessionsCacheEntry deserialize(byte[] bytes) throws SerializationException { + public TransportProtos.DeviceSessionsCacheEntry deserialize(DeviceId key, byte[] bytes) throws SerializationException { try { return TransportProtos.DeviceSessionsCacheEntry.parseFrom(bytes); } catch (InvalidProtocolBufferException e) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index b2abe2a581..2b96ebd4ed 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -34,6 +34,7 @@ import org.thingsboard.server.dao.relation.RelationService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.action.EntityActionService; import org.thingsboard.server.service.apiusage.RateLimitService; +import org.thingsboard.server.service.entitiy.TbNotificationEntityService; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; @@ -57,9 +58,9 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS private final Map> exportServices = new HashMap<>(); private final Map> importServices = new HashMap<>(); - private final EntityActionService entityActionService; private final RelationService relationService; private final RateLimitService rateLimitService; + private final TbNotificationEntityService entityNotificationService; protected static final List SUPPORTED_ENTITY_TYPES = List.of( EntityType.CUSTOMER, EntityType.ASSET, EntityType.RULE_CHAIN, @@ -109,10 +110,8 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations())); for (EntityRelation relation : ctx.getRelations()) { - entityActionService.logEntityAction(ctx.getUser(), relation.getFrom(), null, null, - ActionType.RELATION_ADD_OR_UPDATE, null, relation); - entityActionService.logEntityAction(ctx.getUser(), relation.getTo(), null, null, - ActionType.RELATION_ADD_OR_UPDATE, null, relation); + entityNotificationService.notifyCreateOrUpdateOrDeleteRelation(ctx.getTenantId(), null, + relation, ctx.getUser(), ActionType.RELATION_ADD_OR_UPDATE, null, relation); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java index 169b7f273c..01626368dc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/BaseEntityExportService.java @@ -41,8 +41,6 @@ public abstract class BaseEntityExportService(); } - ; - public abstract Set getSupportedEntityTypes(); protected void replaceUuidsRecursively(EntitiesExportCtx ctx, JsonNode node, Set skipFieldsSet) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java index 7e236370df..a00a7c09aa 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/exporting/impl/DefaultEntityExportService.java @@ -30,7 +30,7 @@ import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.sync.ie.AttributeExportData; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.attributes.AttributesService; -import org.thingsboard.server.dao.relation.RelationService; +import org.thingsboard.server.dao.relation.RelationDao; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.ExportableEntitiesService; @@ -55,7 +55,7 @@ public class DefaultEntityExportService exportRelations(EntitiesExportCtx ctx, E entity) throws ThingsboardException { List relations = new ArrayList<>(); - List inboundRelations = relationService.findByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List inboundRelations = relationDao.findAllByTo(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(inboundRelations); - List outboundRelations = relationService.findByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); + List outboundRelations = relationDao.findAllByFrom(ctx.getTenantId(), entity.getId(), RelationTypeGroup.COMMON); relations.addAll(outboundRelations); return relations; } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java index f2e61b8f2d..3225668004 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/AssetImportService.java @@ -19,14 +19,11 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.edge.EdgeEventActionType; -import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.ie.EntityExportData; import org.thingsboard.server.dao.asset.AssetService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @Service @@ -52,14 +49,6 @@ public class AssetImportService extends BaseEntityImportService existingRelations = new ArrayList<>(); - existingRelations.addAll(relationService.findByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); - existingRelations.addAll(relationService.findByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationDao.findAllByTo(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + existingRelations.addAll(relationDao.findAllByFrom(tenantId, entity.getId(), RelationTypeGroup.COMMON)); + // dao is used here instead of service to avoid getting cached values, because relationService.deleteRelation will evict value from cache only after transaction is committed for (EntityRelation existingRelation : existingRelations) { EntityRelation relation = relationsMap.get(existingRelation); if (relation == null) { importResult.setUpdatedRelatedEntities(true); - relationService.deleteRelation(tenantId, existingRelation); + relationService.deleteRelation(ctx.getTenantId(), existingRelation.getFrom(), existingRelation.getTo(), existingRelation.getType(), existingRelation.getTypeGroup()); importResult.addSendEventsCallback(() -> { - entityActionService.logEntityAction(ctx.getUser(), existingRelation.getFrom(), null, null, - ActionType.RELATION_DELETED, null, existingRelation); - entityActionService.logEntityAction(ctx.getUser(), existingRelation.getTo(), null, null, - ActionType.RELATION_DELETED, null, existingRelation); + entityNotificationService.notifyCreateOrUpdateOrDeleteRelation(tenantId, null, + existingRelation, ctx.getUser(), ActionType.RELATION_DELETED, null, existingRelation); }); } else if (Objects.equal(relation.getAdditionalInfo(), existingRelation.getAdditionalInfo())) { relationsMap.remove(relation); @@ -269,9 +268,8 @@ public abstract class BaseEntityImportService exportData, Dashboard prepared, Dashboard existing) { + return super.compare(ctx, exportData, prepared, existing) || !prepared.getConfiguration().equals(existing.getConfiguration()); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java index c699f466d7..853657d1c2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/importing/impl/DeviceImportService.java @@ -19,6 +19,7 @@ import lombok.RequiredArgsConstructor; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.audit.ActionType; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.TenantId; @@ -93,8 +94,8 @@ public class DeviceImportService extends BaseEntityImportService T getStatus(SecurityUser user, UUID requestId, Function getter) throws ThingsboardException { var cacheEntry = taskCache.get(requestId); if (cacheEntry == null || cacheEntry.get() == null) { + log.debug("[{}] No cache record: {}", requestId, cacheEntry); throw new ThingsboardException(ThingsboardErrorCode.ITEM_NOT_FOUND); } else { var entry = cacheEntry.get(); + log.debug("[{}] Cache get: {}", requestId, entry); var result = getter.apply(entry); if (result == null) { throw new ThingsboardException(ThingsboardErrorCode.BAD_REQUEST_PARAMS); @@ -362,6 +364,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .build(); } + @SneakyThrows @SuppressWarnings({"rawtypes", "unchecked"}) private void importEntities(EntitiesImportCtx ctx, EntityType entityType) { int limit = 100; @@ -373,9 +376,9 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (InterruptedException | ExecutionException e) { throw new RuntimeException(e); } + log.debug("[{}] Loading {} entities pack ({})", ctx.getTenantId(), entityType, entityDataList.size()); for (EntityExportData entityData : entityDataList) { EntityExportData reimportBackup = JacksonUtil.clone(entityData); - log.debug("[{}] Loading {} entities", ctx.getTenantId(), entityType); EntityImportResult importResult; try { importResult = exportImportService.importEntity(ctx, entityData); @@ -391,6 +394,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } + log.debug("Imported {} pack for tenant {}", entityType, ctx.getTenantId()); offset += limit; } while (entityDataList.size() == limit); } @@ -456,18 +460,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont EntityId externalId = ((ExportableEntity) entity).getExternalId(); if (externalId == null) externalId = entityId; - return transformAsync(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), + return transform(gitServiceQueue.getEntity(user.getTenantId(), versionId, externalId), otherVersion -> { SimpleEntitiesExportCtx ctx = new SimpleEntitiesExportCtx(user, null, null, EntityExportSettings.builder() .exportRelations(otherVersion.hasRelations()) .exportAttributes(otherVersion.hasAttributes()) .exportCredentials(otherVersion.hasCredentials()) .build()); - EntityExportData currentVersion = exportImportService.exportEntity(ctx, entityId); - return transform(gitServiceQueue.getContentsDiff(user.getTenantId(), - JacksonUtil.toPrettyString(currentVersion.sort()), - JacksonUtil.toPrettyString(otherVersion.sort())), - rawDiff -> new EntityDataDiff(currentVersion, otherVersion, rawDiff), MoreExecutors.directExecutor()); + EntityExportData currentVersion; + try { + currentVersion = exportImportService.exportEntity(ctx, entityId); + } catch (ThingsboardException e) { + throw new RuntimeException(e); + } + return new EntityDataDiff(currentVersion.sort(), otherVersion.sort()); }, MoreExecutors.directExecutor()); } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java index 628605824e..6eaa2df74f 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultGitVersionControlQueueService.java @@ -15,7 +15,10 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; import com.google.protobuf.ByteString; import lombok.SneakyThrows; @@ -23,6 +26,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.CollectionsUtil; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.EntityType; @@ -71,12 +75,16 @@ import org.thingsboard.server.service.sync.vc.data.VersionsDiffGitRequest; import org.thingsboard.server.service.sync.vc.data.VoidGitRequest; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; import java.util.function.Function; import java.util.stream.Collectors; @@ -93,9 +101,12 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu private final SchedulerComponent scheduler; private final Map> pendingRequestMap = new HashMap<>(); + private final Map> chunkedMsgs = new ConcurrentHashMap<>(); @Value("${queue.vc.request-timeout:60000}") private int requestTimeout; + @Value("${queue.vc.msg-chunk-size:500000}") + private int msgChunkSize; public DefaultGitVersionControlQueueService(TbServiceInfoProvider serviceInfoProvider, TbClusterService clusterService, DataDecodingEncodingService encodingService, @@ -119,20 +130,35 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return future; } + @SuppressWarnings("UnstableApiUsage") @Override public ListenableFuture addToCommit(CommitGitRequest commit, EntityExportData> entityData) { - SettableFuture future = SettableFuture.create(); - String path = getRelativePath(entityData.getEntityType(), entityData.getExternalId()); String entityDataJson = JacksonUtil.toPrettyString(entityData.sort()); - registerAndSend(commit, builder -> builder.setCommitRequest( - buildCommitRequest(commit).setAddMsg( - TransportProtos.AddMsg.newBuilder() - .setRelativePath(path).setEntityDataJson(entityDataJson).build() - ).build() - ).build(), wrap(future, null)); - return future; + Iterable entityDataChunks = StringUtils.split(entityDataJson, msgChunkSize); + String chunkedMsgId = UUID.randomUUID().toString(); + int chunksCount = Iterables.size(entityDataChunks); + + AtomicInteger chunkIndex = new AtomicInteger(); + List> futures = new ArrayList<>(); + entityDataChunks.forEach(chunk -> { + SettableFuture chunkFuture = SettableFuture.create(); + log.trace("[{}] sending chunk {} for 'addToCommit'", chunkedMsgId, chunkIndex.get()); + registerAndSend(commit, builder -> builder.setCommitRequest( + buildCommitRequest(commit).setAddMsg( + TransportProtos.AddMsg.newBuilder() + .setRelativePath(path).setEntityDataJsonChunk(chunk) + .setChunkedMsgId(chunkedMsgId).setChunkIndex(chunkIndex.getAndIncrement()) + .setChunksCount(chunksCount).build() + ).build() + ).build(), wrap(chunkFuture, null)); + futures.add(chunkFuture); + }); + return Futures.transform(Futures.allAsList(futures), r -> { + log.trace("[{}] sent all chunks for 'addToCommit'", chunkedMsgId); + return null; + }, MoreExecutors.directExecutor()); } @Override @@ -221,7 +247,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId, EntityType entityType) { return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() - .setBranchName(branch) .setVersionId(versionId) .setEntityType(entityType.name()) .build()); @@ -230,7 +255,6 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public ListenableFuture> listEntitiesAtVersion(TenantId tenantId, String branch, String versionId) { return listEntitiesAtVersion(tenantId, ListEntitiesRequestMsg.newBuilder() - .setBranchName(branch) .setVersionId(versionId) .build()); } @@ -257,18 +281,11 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .build())); } - @Override - public ListenableFuture getContentsDiff(TenantId tenantId, String content1, String content2) { - ContentsDiffGitRequest request = new ContentsDiffGitRequest(tenantId, content1, content2); - return sendRequest(request, builder -> builder.setContentsDiffRequest(TransportProtos.ContentsDiffRequestMsg.newBuilder() - .setContent1(content1) - .setContent2(content2))); - } - @Override @SuppressWarnings("rawtypes") public ListenableFuture getEntity(TenantId tenantId, String versionId, EntityId entityId) { EntityContentGitRequest request = new EntityContentGitRequest(tenantId, versionId, entityId); + chunkedMsgs.put(request.getRequestId(), new HashMap<>()); registerAndSend(request, builder -> builder.setEntityContentRequest(EntityContentRequestMsg.newBuilder() .setVersionId(versionId) .setEntityType(entityId.getEntityType().name()) @@ -290,9 +307,9 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var requestBody = enrichFunction.apply(newRequestProto(request, settings)); log.trace("[{}][{}] PUSHING request: {}", request.getTenantId(), request.getRequestId(), requestBody); clusterService.pushMsgToVersionControl(request.getTenantId(), requestBody, callback); - request.setTimeoutTask(scheduler.schedule(() -> { - processTimeout(request.getRequestId()); - }, requestTimeout, TimeUnit.MILLISECONDS)); + if (request.getTimeoutTask() == null) { + request.setTimeoutTask(scheduler.schedule(() -> processTimeout(request.getRequestId()), requestTimeout, TimeUnit.MILLISECONDS)); + } } else { throw new RuntimeException("Future is already done!"); } @@ -310,7 +327,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @SuppressWarnings("rawtypes") public ListenableFuture> getEntities(TenantId tenantId, String versionId, EntityType entityType, int offset, int limit) { EntitiesContentGitRequest request = new EntitiesContentGitRequest(tenantId, versionId, entityType); - + chunkedMsgs.put(request.getRequestId(), new HashMap<>()); registerAndSend(request, builder -> builder.setEntitiesContentRequest(EntitiesContentRequestMsg.newBuilder() .setVersionId(versionId) .setEntityType(entityType.name()) @@ -356,15 +373,15 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu @Override public void processResponse(VersionControlResponseMsg vcResponseMsg) { UUID requestId = new UUID(vcResponseMsg.getRequestIdMSB(), vcResponseMsg.getRequestIdLSB()); - PendingGitRequest request = pendingRequestMap.remove(requestId); + PendingGitRequest request = pendingRequestMap.get(requestId); if (request == null) { log.debug("[{}] received stale response: {}", requestId, vcResponseMsg); return; } else { log.debug("[{}] processing response: {}", requestId, vcResponseMsg); - request.getTimeoutTask().cancel(true); } var future = request.getFuture(); + boolean completed = true; if (!StringUtils.isEmpty(vcResponseMsg.getError())) { future.setException(new RuntimeException(vcResponseMsg.getError())); } else { @@ -392,12 +409,28 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu var listVersionsResponse = vcResponseMsg.getListVersionsResponse(); ((ListVersionsGitRequest) request).getFuture().set(toPageData(listVersionsResponse)); } else if (vcResponseMsg.hasEntityContentResponse()) { - var data = vcResponseMsg.getEntityContentResponse().getData(); - ((EntityContentGitRequest) request).getFuture().set(toData(data)); + TransportProtos.EntityContentResponseMsg responseMsg = vcResponseMsg.getEntityContentResponse(); + log.trace("[{}] received chunk {} for 'getEntity'", responseMsg.getChunkedMsgId(), responseMsg.getChunkIndex()); + var joined = joinChunks(requestId, responseMsg, 1); + if (joined.isPresent()) { + log.trace("[{}] collected all chunks for 'getEntity'", responseMsg.getChunkedMsgId()); + ((EntityContentGitRequest) request).getFuture().set(joined.get().get(0)); + } else { + completed = false; + } } else if (vcResponseMsg.hasEntitiesContentResponse()) { - var dataList = vcResponseMsg.getEntitiesContentResponse().getDataList(); - ((EntitiesContentGitRequest) request).getFuture() - .set(dataList.stream().map(this::toData).collect(Collectors.toList())); + TransportProtos.EntitiesContentResponseMsg responseMsg = vcResponseMsg.getEntitiesContentResponse(); + TransportProtos.EntityContentResponseMsg item = responseMsg.getItem(); + if (responseMsg.getItemsCount() > 0) { + var joined = joinChunks(requestId, item, responseMsg.getItemsCount()); + if (joined.isPresent()) { + ((EntitiesContentGitRequest) request).getFuture().set(joined.get()); + } else { + completed = false; + } + } else { + ((EntitiesContentGitRequest) request).getFuture().set(Collections.emptyList()); + } } else if (vcResponseMsg.hasVersionsDiffResponse()) { TransportProtos.VersionsDiffResponseMsg diffResponse = vcResponseMsg.getVersionsDiffResponse(); List entityVersionsDiffList = diffResponse.getDiffList().stream() @@ -412,21 +445,50 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu .build()) .collect(Collectors.toList()); ((VersionsDiffGitRequest) request).getFuture().set(entityVersionsDiffList); - } else if (vcResponseMsg.hasContentsDiffResponse()) { - String diff = vcResponseMsg.getContentsDiffResponse().getDiff(); - ((ContentsDiffGitRequest) request).getFuture().set(diff); } } + if (completed) { + removePendingRequest(requestId); + } + } + + @SuppressWarnings("rawtypes") + private Optional> joinChunks(UUID requestId, TransportProtos.EntityContentResponseMsg responseMsg, int expectedMsgCount) { + var chunksMap = chunkedMsgs.get(requestId); + if (chunksMap == null) { + return Optional.empty(); + } + String[] msgChunks = chunksMap.computeIfAbsent(responseMsg.getChunkedMsgId(), id -> new String[responseMsg.getChunksCount()]); + msgChunks[responseMsg.getChunkIndex()] = responseMsg.getData(); + if (chunksMap.size() == expectedMsgCount && chunksMap.values().stream() + .allMatch(chunks -> CollectionsUtil.countNonNull(chunks) == chunks.length)) { + return Optional.of(chunksMap.values().stream() + .map(chunks -> String.join("", chunks)) + .map(this::toData) + .collect(Collectors.toList())); + } else { + return Optional.empty(); + } } private void processTimeout(UUID requestId) { - PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + PendingGitRequest pendingRequest = removePendingRequest(requestId); if (pendingRequest != null) { log.debug("[{}] request timed out ({} ms}", requestId, requestTimeout); pendingRequest.getFuture().setException(new TimeoutException("Request timed out")); } } + private PendingGitRequest removePendingRequest(UUID requestId) { + PendingGitRequest pendingRequest = pendingRequestMap.remove(requestId); + if (pendingRequest != null && pendingRequest.getTimeoutTask() != null) { + pendingRequest.getTimeoutTask().cancel(true); + pendingRequest.setTimeoutTask(null); + } + chunkedMsgs.remove(requestId); + return pendingRequest; + } + private PageData toPageData(TransportProtos.ListVersionsResponseMsg listVersionsResponse) { var listVersions = listVersionsResponse.getVersionsList().stream().map(this::getEntityVersion).collect(Collectors.toList()); return new PageData<>(listVersions, listVersionsResponse.getTotalPages(), listVersionsResponse.getTotalElements(), listVersionsResponse.getHasNext()); @@ -450,6 +512,7 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu return JacksonUtil.fromString(data, EntityExportData.class); } + //The future will be completed when the corresponding result arrives from kafka private static TbQueueCallback wrap(SettableFuture future) { return new TbQueueCallback() { @Override @@ -463,7 +526,8 @@ public class DefaultGitVersionControlQueueService implements GitVersionControlQu }; } - private static TbQueueCallback wrap(SettableFuture future, T value) { + //The future will be completed when the request is successfully sent to kafka + private TbQueueCallback wrap(SettableFuture future, T value) { return new TbQueueCallback() { @Override public void onSuccess(TbQueueMsgMetadata metadata) { diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java index a1e64da8f1..cc83479896 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/GitVersionControlQueueService.java @@ -64,8 +64,6 @@ public interface GitVersionControlQueueService { ListenableFuture> getVersionsDiff(TenantId tenantId, EntityType entityType, EntityId externalId, String versionId1, String versionId2); - ListenableFuture getContentsDiff(TenantId tenantId, String rawEntityData1, String rawEntityData2); - ListenableFuture initRepository(TenantId tenantId, RepositorySettings settings); ListenableFuture testRepository(TenantId tenantId, RepositorySettings settings); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskRedisCache.java index be053f6308..6215744f7a 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/VersionControlTaskRedisCache.java @@ -15,19 +15,14 @@ */ package org.thingsboard.server.service.sync.vc; -import com.google.protobuf.InvalidProtocolBufferException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.serializer.RedisSerializer; -import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.gen.transport.TransportProtos; import java.util.UUID; @@ -36,6 +31,6 @@ import java.util.UUID; public class VersionControlTaskRedisCache extends RedisTbTransactionalCache { public VersionControlTaskRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.VERSION_CONTROL_TASK_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.VERSION_CONTROL_TASK_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java index f88b1cf6bd..e3c918818c 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/autocommit/AutoCommitSettingsRedisCache.java @@ -21,7 +21,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; @@ -31,6 +31,6 @@ import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; public class AutoCommitSettingsRedisCache extends RedisTbTransactionalCache { public AutoCommitSettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.AUTO_COMMIT_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java index 3cccb6d24e..a49ace3ec2 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/repository/RepositorySettingsRedisCache.java @@ -21,7 +21,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; @@ -31,6 +31,6 @@ import org.thingsboard.server.common.data.sync.vc.RepositorySettings; public class RepositorySettingsRedisCache extends RedisTbTransactionalCache { public RepositorySettingsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.REPOSITORY_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.REPOSITORY_SETTINGS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 995ebb43c4..4784ad2d43 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,26 +25,31 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - + - - + + diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 07233bb241..766b393e78 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -428,6 +428,9 @@ cache: timeToLiveInMinutes: "${CACHE_SPECS_VERSION_CONTROL_TASK_TTL:5}" maxSize: "${CACHE_SPECS_VERSION_CONTROL_TASK_MAX_SIZE:100000}" +#Disable this because it is not required. +spring.data.redis.repositories.enabled: false + redis: # standalone or cluster connection: @@ -1024,6 +1027,7 @@ queue: poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" request-timeout: "${TB_QUEUE_VC_REQUEST_TIMEOUT:60000}" + msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:500000}" js: # JS Eval request topic request_topic: "${REMOTE_JS_EVAL_REQUEST_TOPIC:js_eval.requests}" diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index b3ab39fd51..ce43ae3bb5 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -16,8 +16,6 @@ package org.thingsboard.server.transport.coap; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.junit.After; import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.Device; @@ -41,7 +39,6 @@ import org.thingsboard.server.common.data.device.profile.JsonTransportPayloadCon import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.security.DeviceCredentials; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.AbstractTransportIntegrationTest; import static org.junit.Assert.assertEquals; @@ -54,12 +51,11 @@ import static org.junit.Assert.assertNotNull; public abstract class AbstractCoapIntegrationTest extends AbstractTransportIntegrationTest { protected final byte[] EMPTY_PAYLOAD = new byte[0]; - - protected CoapClient client; + protected CoapTestClient client; protected void processAfterTest() throws Exception { if (client != null) { - client.shutdown(); + client.disconnect(); } } @@ -156,16 +152,4 @@ public abstract class AbstractCoapIntegrationTest extends AbstractTransportInteg device.setType(type); return doPost("/api/device", device, Device.class); } - - protected CoapClient getCoapClient(FeatureType featureType) { - return new CoapClient(getFeatureTokenUrl(accessToken, featureType)); - } - - protected CoapClient getCoapClient(String featureTokenUrl) { - return new CoapClient(featureTokenUrl); - } - - protected String getFeatureTokenUrl(String token, FeatureType featureType) { - return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase(); - } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java new file mode 100644 index 0000000000..89fa13220d --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestCallback.java @@ -0,0 +1,68 @@ +/** + * Copyright ยฉ 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.eclipse.californium.core.CoapHandler; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; + +import java.util.concurrent.CountDownLatch; + +@Slf4j +@Data +public class CoapTestCallback implements CoapHandler { + + protected final CountDownLatch latch; + protected Integer observe; + protected byte[] payloadBytes; + protected CoAP.ResponseCode responseCode; + + public CoapTestCallback() { + this.latch = new CountDownLatch(1); + } + + public CoapTestCallback(int subscribeCount) { + this.latch = new CountDownLatch(subscribeCount); + } + + public Integer getObserve() { + return observe; + } + + public byte[] getPayloadBytes() { + return payloadBytes; + } + + public CoAP.ResponseCode getResponseCode() { + return responseCode; + } + + @Override + public void onLoad(CoapResponse response) { + observe = response.getOptions().getObserve(); + payloadBytes = response.getPayload(); + responseCode = response.getCode(); + latch.countDown(); + } + + @Override + public void onError() { + log.warn("Command Response Ack Error, No connect"); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java new file mode 100644 index 0000000000..dc2a3f4c83 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/transport/coap/CoapTestClient.java @@ -0,0 +1,118 @@ +/** + * Copyright ยฉ 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.coap; + +import org.eclipse.californium.core.CoapClient; +import org.eclipse.californium.core.CoapHandler; +import org.eclipse.californium.core.CoapObserveRelation; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; +import org.eclipse.californium.core.coap.MediaTypeRegistry; +import org.eclipse.californium.core.coap.Request; +import org.eclipse.californium.elements.exception.ConnectorException; +import org.thingsboard.server.common.msg.session.FeatureType; + +import java.io.IOException; + +public class CoapTestClient { + + private static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/"; + private static final long CLIENT_REQUEST_TIMEOUT = 60000L; + + private final CoapClient client; + + public CoapTestClient(){ + this.client = createClient(); + } + + public CoapTestClient(String accessToken, FeatureType featureType) { + this.client = createClient(getFeatureTokenUrl(accessToken, featureType)); + } + + public CoapTestClient(String featureTokenUrl) { + this.client = createClient(featureTokenUrl); + } + + public void connectToCoap(String accessToken) { + setURI(accessToken, null); + } + + public void connectToCoap(String accessToken, FeatureType featureType) { + setURI(accessToken, featureType); + } + + public void disconnect() { + if (client != null) { + client.shutdown(); + } + } + + public CoapResponse postMethod(String requestBody) throws ConnectorException, IOException { + return this.postMethod(requestBody.getBytes()); + } + + public CoapResponse postMethod(byte[] requestBodyBytes) throws ConnectorException, IOException { + return client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(requestBodyBytes, MediaTypeRegistry.APPLICATION_JSON); + } + + public void postMethod(CoapHandler handler, String payload, int format) { + client.post(handler, payload, format); + } + + public void postMethod(CoapHandler handler, byte[] payload, int format) { + client.post(handler, payload, format); + } + + public CoapResponse getMethod() throws ConnectorException, IOException { + return client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); + } + + public CoapObserveRelation getObserveRelation(CoapTestCallback callback){ + Request request = Request.newGet().setObserve(); + request.setType(CoAP.Type.CON); + return client.observe(request, callback); + } + + public void setURI(String featureTokenUrl) { + if (client == null) { + throw new RuntimeException("Failed to connect! CoapClient is not initialized!"); + } + client.setURI(featureTokenUrl); + } + + public void setURI(String accessToken, FeatureType featureType) { + if (featureType == null){ + featureType = FeatureType.ATTRIBUTES; + } + setURI(getFeatureTokenUrl(accessToken, featureType)); + } + + private CoapClient createClient() { + return new CoapClient(); + } + + private CoapClient createClient(String featureTokenUrl) { + return new CoapClient(featureTokenUrl); + } + + public static String getFeatureTokenUrl(String token, FeatureType featureType) { + return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase(); + } + + public static String getFeatureTokenUrl(String token, FeatureType featureType, int requestId) { + return COAP_BASE_URL + token + "/" + featureType.name().toLowerCase() + "/" + requestId; + } +} diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java index 9d0078aa3f..9710c82866 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/AbstractCoapAttributesIntegrationTest.java @@ -15,25 +15,94 @@ */ package org.thingsboard.server.transport.coap.attributes; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; +import org.eclipse.californium.core.CoapObserveRelation; +import org.eclipse.californium.core.CoapResponse; +import org.eclipse.californium.core.coap.CoAP; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.SingleEntityFilter; +import org.thingsboard.server.common.msg.session.FeatureType; +import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestCallback; +import org.thingsboard.server.transport.coap.CoapTestClient; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.common.data.query.EntityKeyType.CLIENT_ATTRIBUTE; +import static org.thingsboard.server.common.data.query.EntityKeyType.SHARED_ATTRIBUTE; @Slf4j public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { - protected static final String POST_ATTRIBUTES_PAYLOAD = "{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73," + - "\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + @Autowired + DefaultTransportService defaultTransportService; + + public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + + "\n" + + "package test;\n" + + "\n" + + "message PostAttributes {\n" + + " string clientStr = 1;\n" + + " bool clientBool = 2;\n" + + " double clientDbl = 3;\n" + + " int32 clientLong = 4;\n" + + " JsonObject clientJson = 5;\n" + + "\n" + + " message JsonObject {\n" + + " int32 someNumber = 6;\n" + + " repeated int32 someArray = 7;\n" + + " NestedJsonObject someNestedObject = 8;\n" + + " message NestedJsonObject {\n" + + " string key = 9;\n" + + " }\n" + + " }\n" + + "}"; + + private static final String CLIENT_ATTRIBUTES_PAYLOAD = "{\"clientStr\":\"value1\",\"clientBool\":true,\"clientDbl\":42.0,\"clientLong\":73," + + "\"clientJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + + private static final String SHARED_ATTRIBUTES_PAYLOAD = "{\"sharedStr\":\"value1\",\"sharedBool\":true,\"sharedDbl\":42.0,\"sharedLong\":73," + + "\"sharedJson\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}"; + + protected static final String SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"sharedStr\":\"value\",\"sharedBool\":false,\"sharedDbl\":41.0,\"sharedLong\":72," + + "\"sharedJson\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}"; - protected List getTsKvProtoList() { - TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value1", TransportProtos.KeyValueType.STRING_V); - TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "true", TransportProtos.KeyValueType.BOOLEAN_V); - TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "42.0", TransportProtos.KeyValueType.DOUBLE_V); - TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "73", TransportProtos.KeyValueType.LONG_V); - TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); + private static final String SHARED_ATTRIBUTES_DELETED_RESPONSE = "{\"deleted\":[\"sharedJson\"]}"; + + private List getTsKvProtoList(String attributePrefix) { + TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto(attributePrefix + "Str", "value1", TransportProtos.KeyValueType.STRING_V); + TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto(attributePrefix + "Bool", "true", TransportProtos.KeyValueType.BOOLEAN_V); + TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto(attributePrefix + "Dbl", "42.0", TransportProtos.KeyValueType.DOUBLE_V); + TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto(attributePrefix + "Long", "73", TransportProtos.KeyValueType.LONG_V); + TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto(attributePrefix + "Json", "{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); List tsKvProtoList = new ArrayList<>(); tsKvProtoList.add(tsKvProtoAttribute1); tsKvProtoList.add(tsKvProtoAttribute2); @@ -49,4 +118,295 @@ public abstract class AbstractCoapAttributesIntegrationTest extends AbstractCoap tsKvProtoBuilder.setKv(keyValueProto); return tsKvProtoBuilder.build(); } + + private List getEntityKeys(List keys, EntityKeyType scope) { + return keys.stream().map(key -> new EntityKey(scope, key)).collect(Collectors.toList()); + } + + private byte[] getAttributesProtoPayloadBytes() { + + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); + DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); + + DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); + Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); + assertNotNull(nestedJsonObjectBuilderDescriptor); + DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); + + DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); + Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); + assertNotNull(jsonObjectBuilderDescriptor); + DynamicMessage jsonObject = jsonObjectBuilder + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) + .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) + .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) + .build(); + + DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); + Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); + assertNotNull(postAttributesMsgDescriptor); + DynamicMessage postAttributesMsg = postAttributesBuilder + .setField(postAttributesMsgDescriptor.findFieldByName("clientStr"), "value1") + .setField(postAttributesMsgDescriptor.findFieldByName("clientBool"), true) + .setField(postAttributesMsgDescriptor.findFieldByName("clientDbl"), 42.0) + .setField(postAttributesMsgDescriptor.findFieldByName("clientLong"), 73) + .setField(postAttributesMsgDescriptor.findFieldByName("clientJson"), jsonObject) + .build(); + return postAttributesMsg.toByteArray(); + } + + protected void processJsonTestRequestAttributesValuesFromTheServer() throws Exception { + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + SingleEntityFilter dtf = new SingleEntityFilter(); + dtf.setSingleEntity(savedDevice.getId()); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; + List clientKeysList = List.of(clientKeysStr.split(",")); + List sharedKeysList = List.of(sharedKeysStr.split(",")); + List csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE); + List shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE); + List keys = new ArrayList<>(); + keys.addAll(csKeys); + keys.addAll(shKeys); + getWsClient().subscribeLatestUpdate(keys, dtf); + getWsClient().registerWaitForUpdate(2); + + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", + SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + + CoapResponse coapResponse = client.postMethod(CLIENT_ATTRIBUTES_PAYLOAD); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + + String update = getWsClient().waitForUpdate(); + assertThat(update).as("ws update received").isNotBlank(); + + String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; + client.setURI(featureTokenUrl); + validateJsonResponse(client.getMethod()); + } + + protected void processProtoTestRequestAttributesValuesFromTheServer() throws Exception { + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + SingleEntityFilter dtf = new SingleEntityFilter(); + dtf.setSingleEntity(savedDevice.getId()); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; + List clientKeysList = List.of(clientKeysStr.split(",")); + List sharedKeysList = List.of(sharedKeysStr.split(",")); + List csKeys = getEntityKeys(clientKeysList, CLIENT_ATTRIBUTE); + List shKeys = getEntityKeys(sharedKeysList, SHARED_ATTRIBUTE); + List keys = new ArrayList<>(); + keys.addAll(csKeys); + keys.addAll(shKeys); + getWsClient().subscribeLatestUpdate(keys, dtf); + getWsClient().registerWaitForUpdate(2); + + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", + SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + + CoapResponse coapResponse = client.postMethod(getAttributesProtoPayloadBytes()); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); + + String update = getWsClient().waitForUpdate(); + assertThat(update).as("ws update received").isNotBlank(); + + String featureTokenUrl = CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + clientKeysStr + "&sharedKeys=" + sharedKeysStr; + client.setURI(featureTokenUrl); + validateProtoResponse(client.getMethod()); + } + + protected void processJsonTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { + if (!emptyCurrentStateNotification) { + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); + } + + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapTestCallback callbackCoap = new CoapTestCallback(1); + + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + + if (emptyCurrentStateNotification) { + validateUpdateAttributesJsonResponse(callbackCoap, "{}", 0); + } else { + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, 0); + } + + CountDownLatch latch = new CountDownLatch(1); + int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1; + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + latch.await(3, TimeUnit.SECONDS); + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_PAYLOAD, expectedObserveCnt); + + latch = new CountDownLatch(1); + int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1; + doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); + latch.await(3, TimeUnit.SECONDS); + validateUpdateAttributesJsonResponse(callbackCoap, SHARED_ATTRIBUTES_DELETED_RESPONSE, expectedObserveBeforeDeleteCnt); + + observeRelation.proactiveCancel(); + assertTrue(observeRelation.isCanceled()); + + awaitClientAfterCancelObserve(); + } + + protected void processProtoTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { + if (!emptyCurrentStateNotification) { + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); + } + + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapTestCallback callbackCoap = new CoapTestCallback(1); + + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + + if (emptyCurrentStateNotification) { + validateEmptyCurrentStateAttributesProtoResponse(callbackCoap); + } else { + validateCurrentStateAttributesProtoResponse(callbackCoap); + } + + CountDownLatch latch = new CountDownLatch(1); + int expectedObserveCnt = callbackCoap.getObserve().intValue() + 1; + doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); + latch.await(3, TimeUnit.SECONDS); + validateUpdateProtoAttributesResponse(callbackCoap, expectedObserveCnt); + + latch = new CountDownLatch(1); + int expectedObserveBeforeDeleteCnt = callbackCoap.getObserve().intValue() + 1; + doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); + latch.await(3, TimeUnit.SECONDS); + validateDeleteProtoAttributesResponse(callbackCoap, expectedObserveBeforeDeleteCnt); + + observeRelation.proactiveCancel(); + assertTrue(observeRelation.isCanceled()); + + awaitClientAfterCancelObserve(); + } + + protected void validateJsonResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { + assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); + String expectedResponse = "{\"client\":" + CLIENT_ATTRIBUTES_PAYLOAD + ",\"shared\":" + SHARED_ATTRIBUTES_PAYLOAD + "}"; + assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(getAttributesResponse.getPayload())); + } + + protected void validateProtoResponse(CoapResponse getAttributesResponse) throws InterruptedException, InvalidProtocolBufferException { + TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); + TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); + assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); + List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); + assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); + } + + protected void validateUpdateAttributesJsonResponse(CoapTestCallback callback, String expectedResponse, int expectedObserveCnt) { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); + assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.toJsonNode(response)); + } + + protected void validateEmptyCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException { + assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(0, callback.getObserve().intValue()); + } + + protected void validateCurrentStateAttributesProtoResponse(CoapTestCallback callback) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(0, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("sharedStr", "value", TransportProtos.KeyValueType.STRING_V); + TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("sharedBool", "false", TransportProtos.KeyValueType.BOOLEAN_V); + TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("sharedDbl", "41.0", TransportProtos.KeyValueType.DOUBLE_V); + TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("sharedLong", "72", TransportProtos.KeyValueType.LONG_V); + TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("sharedJson", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); + List tsKvProtoList = new ArrayList<>(); + tsKvProtoList.add(tsKvProtoAttribute1); + tsKvProtoList.add(tsKvProtoAttribute2); + tsKvProtoList.add(tsKvProtoAttribute3); + tsKvProtoList.add(tsKvProtoAttribute4); + tsKvProtoList.add(tsKvProtoAttribute5); + TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build(); + TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + List expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + + assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); + assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); + } + + protected void validateUpdateProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + List tsKvProtoList = getTsKvProtoList("shared"); + attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); + + TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); + TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + List actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + List expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); + + assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); + assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); + } + + protected void validateDeleteProtoAttributesResponse(CoapTestCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveCnt, callback.getObserve().intValue()); + TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); + attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); + + TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); + TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); + + assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size()); + assertEquals("sharedJson", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0)); + } + + private void awaitClientAfterCancelObserve() { + Awaitility.await("awaitClientAfterCancelObserve") + .pollInterval(10, TimeUnit.MILLISECONDS) + .atMost(5, TimeUnit.SECONDS) + .until(()->{ + log.trace("awaiting defaultTransportService.sessions is empty"); + return defaultTransportService.sessions.isEmpty();}); + } + + private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { + TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); + List csTsKvProtoList = getTsKvProtoList("client"); + List shTsKvProtoList = getTsKvProtoList("shared"); + result.addAllClientAttributeList(csTsKvProtoList); + result.addAllSharedAttributeList(shTsKvProtoList); + result.setRequestId(0); + return result.build(); + } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java index 977502e0e6..727fb52380 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestIntegrationTest.java @@ -15,34 +15,18 @@ */ package org.thingsboard.server.transport.coap.attributes.request; -import com.fasterxml.jackson.core.type.TypeReference; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.After; import org.junit.Before; import org.junit.Test; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; -import org.thingsboard.server.common.msg.session.FeatureType; - -import java.nio.charset.StandardCharsets; -import java.util.List; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j @DaoSqlTest public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributesIntegrationTest { - protected static final long CLIENT_REQUEST_TIMEOUT = 60000L; - @Before public void beforeTest() throws Exception { CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() @@ -58,44 +42,6 @@ public class CoapAttributesRequestIntegrationTest extends AbstractCoapAttributes @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - processTestRequestAttributesValuesFromTheServer(); - } - - protected void processTestRequestAttributesValuesFromTheServer() throws Exception { - postAttributes(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; - - List savedAttributeKeys = null; - while (start <= end) { - savedAttributeKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); - if (savedAttributeKeys.size() == 5) { - break; - } - Thread.sleep(100); - start += 100; - } - assertNotNull(savedAttributeKeys); - - String keys = "attribute1,attribute2,attribute3,attribute4,attribute5"; - String featureTokenUrl = getFeatureTokenUrl(accessToken, FeatureType.ATTRIBUTES) + "?clientKeys=" + keys + "&sharedKeys=" + keys; - client = getCoapClient(featureTokenUrl); - - CoapResponse getAttributesResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).get(); - validateResponse(getAttributesResponse); - } - - protected void postAttributes() throws Exception { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - client = getCoapClient(FeatureType.ATTRIBUTES); - CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(POST_ATTRIBUTES_PAYLOAD.getBytes(), MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - - protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { - assertEquals(CoAP.ResponseCode.CONTENT, getAttributesResponse.getCode()); - String expectedRequestPayload = "{\"client\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}},\"shared\":{\"attribute1\":\"value1\",\"attribute2\":true,\"attribute3\":42.0,\"attribute4\":73,\"attribute5\":{\"someNumber\":42,\"someArray\":[1,2,3],\"someNestedObject\":{\"key\":\"value\"}}}}"; - assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.toJsonNode(new String(getAttributesResponse.getPayload(), StandardCharsets.UTF_8))); + processJsonTestRequestAttributesValuesFromTheServer(); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java index e421fbf1a0..2c6928b8d4 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestJsonIntegrationTest.java @@ -45,6 +45,6 @@ public class CoapAttributesRequestJsonIntegrationTest extends CoapAttributesRequ @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - super.testRequestAttributesValuesFromTheServer(); + processJsonTestRequestAttributesValuesFromTheServer(); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java index 7bdcce1400..5f300ba68e 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/request/CoapAttributesRequestProtoIntegrationTest.java @@ -15,64 +15,18 @@ */ package org.thingsboard.server.transport.coap.attributes.request; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - @Slf4j @DaoSqlTest public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesRequestIntegrationTest { - public static final String ATTRIBUTES_SCHEMA_STR = "syntax =\"proto3\";\n" + - "\n" + - "package test;\n" + - "\n" + - "message PostAttributes {\n" + - " string attribute1 = 1;\n" + - " bool attribute2 = 2;\n" + - " double attribute3 = 3;\n" + - " int32 attribute4 = 4;\n" + - " JsonObject attribute5 = 5;\n" + - "\n" + - " message JsonObject {\n" + - " int32 someNumber = 6;\n" + - " repeated int32 someArray = 7;\n" + - " NestedJsonObject someNestedObject = 8;\n" + - " message NestedJsonObject {\n" + - " string key = 9;\n" + - " }\n" + - " }\n" + - "}"; - @Before @Override public void beforeTest() throws Exception { @@ -87,74 +41,6 @@ public class CoapAttributesRequestProtoIntegrationTest extends CoapAttributesReq @Test public void testRequestAttributesValuesFromTheServer() throws Exception { - processTestRequestAttributesValuesFromTheServer(); + processProtoTestRequestAttributesValuesFromTheServer(); } - - protected void postAttributes() throws Exception { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; - ProtoFileElement transportProtoSchema = protoTransportPayloadConfiguration.getTransportProtoSchema(ATTRIBUTES_SCHEMA_STR); - DynamicSchema attributesSchema = protoTransportPayloadConfiguration.getDynamicSchema(transportProtoSchema, ProtoTransportPayloadConfiguration.ATTRIBUTES_PROTO_SCHEMA); - - DynamicMessage.Builder nestedJsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject.NestedJsonObject"); - Descriptors.Descriptor nestedJsonObjectBuilderDescriptor = nestedJsonObjectBuilder.getDescriptorForType(); - assertNotNull(nestedJsonObjectBuilderDescriptor); - DynamicMessage nestedJsonObject = nestedJsonObjectBuilder.setField(nestedJsonObjectBuilderDescriptor.findFieldByName("key"), "value").build(); - - DynamicMessage.Builder jsonObjectBuilder = attributesSchema.newMessageBuilder("PostAttributes.JsonObject"); - Descriptors.Descriptor jsonObjectBuilderDescriptor = jsonObjectBuilder.getDescriptorForType(); - assertNotNull(jsonObjectBuilderDescriptor); - DynamicMessage jsonObject = jsonObjectBuilder - .setField(jsonObjectBuilderDescriptor.findFieldByName("someNumber"), 42) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 1) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 2) - .addRepeatedField(jsonObjectBuilderDescriptor.findFieldByName("someArray"), 3) - .setField(jsonObjectBuilderDescriptor.findFieldByName("someNestedObject"), nestedJsonObject) - .build(); - - DynamicMessage.Builder postAttributesBuilder = attributesSchema.newMessageBuilder("PostAttributes"); - Descriptors.Descriptor postAttributesMsgDescriptor = postAttributesBuilder.getDescriptorForType(); - assertNotNull(postAttributesMsgDescriptor); - DynamicMessage postAttributesMsg = postAttributesBuilder - .setField(postAttributesMsgDescriptor.findFieldByName("attribute1"), "value1") - .setField(postAttributesMsgDescriptor.findFieldByName("attribute2"), true) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute3"), 42.0) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute4"), 73) - .setField(postAttributesMsgDescriptor.findFieldByName("attribute5"), jsonObject) - .build(); - byte[] payload = postAttributesMsg.toByteArray(); - client = getCoapClient(FeatureType.ATTRIBUTES); - CoapResponse coapResponse = client.setTimeout(CLIENT_REQUEST_TIMEOUT).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - - protected void validateResponse(CoapResponse getAttributesResponse) throws InvalidProtocolBufferException { - TransportProtos.GetAttributeResponseMsg expectedAttributesResponse = getExpectedAttributeResponseMsg(); - TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(getAttributesResponse.getPayload()); - assertEquals(expectedAttributesResponse.getRequestId(), actualAttributesResponse.getRequestId()); - List expectedClientKeyValueProtos = expectedAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List expectedSharedKeyValueProtos = expectedAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualClientKeyValueProtos = actualAttributesResponse.getClientAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualSharedKeyValueProtos = actualAttributesResponse.getSharedAttributeListList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - assertTrue(actualClientKeyValueProtos.containsAll(expectedClientKeyValueProtos)); - assertTrue(actualSharedKeyValueProtos.containsAll(expectedSharedKeyValueProtos)); - } - - private TransportProtos.GetAttributeResponseMsg getExpectedAttributeResponseMsg() { - TransportProtos.GetAttributeResponseMsg.Builder result = TransportProtos.GetAttributeResponseMsg.newBuilder(); - List tsKvProtoList = getTsKvProtoList(); - result.addAllClientAttributeList(tsKvProtoList); - result.addAllSharedAttributeList(tsKvProtoList); - result.setRequestId(0); - return result.build(); - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java index bbbfad3c65..3214ae1639 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesIntegrationTest.java @@ -15,47 +15,25 @@ */ package org.thingsboard.server.transport.coap.attributes.updates; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.awaitility.Awaitility; -import org.eclipse.californium.core.CoapHandler; -import org.eclipse.californium.core.CoapObserveRelation; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.Request; import org.eclipse.californium.core.server.resources.Resource; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.coapserver.DefaultCoapServerService; -import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.common.transport.service.DefaultTransportService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import org.thingsboard.server.transport.coap.CoapTransportResource; import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; -import java.nio.charset.StandardCharsets; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.spy; -import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @Slf4j @DaoSqlTest public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributesIntegrationTest { - private static final String RESPONSE_ATTRIBUTES_PAYLOAD_DELETED = "{\"deleted\":[\"attribute5\"]}"; - - protected static final String POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION = "{\"attribute1\":\"value\",\"attribute2\":false,\"attribute3\":41.0,\"attribute4\":72," + - "\"attribute5\":{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}}"; - CoapTransportResource coapTransportResource; @Autowired @@ -83,136 +61,11 @@ public class CoapAttributesUpdatesIntegrationTest extends AbstractCoapAttributes @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - processTestSubscribeToAttributesUpdates(false); + processJsonTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - processTestSubscribeToAttributesUpdates(true); - } - - protected void processTestSubscribeToAttributesUpdates(boolean emptyCurrentStateNotification) throws Exception { - if (!emptyCurrentStateNotification) { - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION, String.class, status().isOk()); - } - client = getCoapClient(FeatureType.ATTRIBUTES); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(latch); - - Request request = Request.newGet().setObserve(); - request.setType(CoAP.Type.CON); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); - - if (emptyCurrentStateNotification) { - validateEmptyCurrentStateAttributesResponse(callback); - } else { - validateCurrentStateAttributesResponse(callback); - } - - latch = new CountDownLatch(1); - int expectedObserveCnt = callback.getObserve().intValue() + 1; - doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", POST_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateUpdateAttributesResponse(callback, expectedObserveCnt); - - - latch = new CountDownLatch(1); - int expectedObserveBeforeDeleteCnt = callback.getObserve().intValue() + 1; - doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=attribute5", String.class); - latch.await(3, TimeUnit.SECONDS); - - validateDeleteAttributesResponse(callback, expectedObserveBeforeDeleteCnt); - observeRelation.proactiveCancel(); - assertTrue(observeRelation.isCanceled()); - - awaitClientAfterCancelObserve(); - } - - protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD_ON_CURRENT_STATE_NOTIFICATION), JacksonUtil.toJsonNode(response)); - } - - protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals("{}", response); - } - - protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(POST_ATTRIBUTES_PAYLOAD), JacksonUtil.toJsonNode(response)); - } - - protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - String response = new String(callback.getPayloadBytes(), StandardCharsets.UTF_8); - assertEquals(JacksonUtil.toJsonNode(RESPONSE_ATTRIBUTES_PAYLOAD_DELETED), JacksonUtil.toJsonNode(response)); - } - - protected static class TestCoapCallback implements CoapHandler { - - private final CountDownLatch latch; - - private Integer observe; - private byte[] payloadBytes; - private CoAP.ResponseCode responseCode; - - public Integer getObserve() { - return observe; - } - - public byte[] getPayloadBytes() { - return payloadBytes; - } - - public CoAP.ResponseCode getResponseCode() { - return responseCode; - } - - private TestCoapCallback(CountDownLatch latch) { - this.latch = latch; - } - - @Override - public void onLoad(CoapResponse response) { - observe = response.getOptions().getObserve(); - payloadBytes = response.getPayload(); - responseCode = response.getCode(); - latch.countDown(); - } - - @Override - public void onError() { - log.warn("Command Response Ack Error, No connect"); - } - - } - - private void awaitClientAfterCancelObserve() { - Awaitility.await("awaitClientAfterCancelObserve") - .pollInterval(10, TimeUnit.MILLISECONDS) - .atMost(5, TimeUnit.SECONDS) - .until(()->{ - log.trace("awaiting defaultTransportService.sessions is empty"); - return defaultTransportService.sessions.isEmpty();}); + processJsonTestSubscribeToAttributesUpdates(true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java index 4ab052db46..816ffa53b3 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesJsonIntegrationTest.java @@ -23,10 +23,11 @@ import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; +import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; @Slf4j @DaoSqlTest -public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpdatesIntegrationTest { +public class CoapAttributesUpdatesJsonIntegrationTest extends AbstractCoapAttributesIntegrationTest { @Before public void beforeTest() throws Exception { @@ -45,11 +46,11 @@ public class CoapAttributesUpdatesJsonIntegrationTest extends CoapAttributesUpda @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - super.testSubscribeToAttributesUpdatesFromTheServer(); + processJsonTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - super.testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification(); + processJsonTestSubscribeToAttributesUpdates(true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java index 4d9970b2dc..4a5e7c033f 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/attributes/updates/CoapAttributesUpdatesProtoIntegrationTest.java @@ -15,30 +15,19 @@ */ package org.thingsboard.server.transport.coap.attributes.updates; -import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.coap.CoAP; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; -import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertTrue; +import org.thingsboard.server.transport.coap.attributes.AbstractCoapAttributesIntegrationTest; @Slf4j @DaoSqlTest -public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpdatesIntegrationTest { +public class CoapAttributesUpdatesProtoIntegrationTest extends AbstractCoapAttributesIntegrationTest { @Before public void beforeTest() throws Exception { @@ -57,83 +46,11 @@ public class CoapAttributesUpdatesProtoIntegrationTest extends CoapAttributesUpd @Test public void testSubscribeToAttributesUpdatesFromTheServer() throws Exception { - processTestSubscribeToAttributesUpdates(false); + processProtoTestSubscribeToAttributesUpdates(false); } @Test public void testSubscribeToAttributesUpdatesFromTheServerWithEmptyCurrentStateNotification() throws Exception { - processTestSubscribeToAttributesUpdates(true); + processProtoTestSubscribeToAttributesUpdates(true); } - - protected void validateCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder expectedCurrentStateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - TransportProtos.TsKvProto tsKvProtoAttribute1 = getTsKvProto("attribute1", "value", TransportProtos.KeyValueType.STRING_V); - TransportProtos.TsKvProto tsKvProtoAttribute2 = getTsKvProto("attribute2", "false", TransportProtos.KeyValueType.BOOLEAN_V); - TransportProtos.TsKvProto tsKvProtoAttribute3 = getTsKvProto("attribute3", "41.0", TransportProtos.KeyValueType.DOUBLE_V); - TransportProtos.TsKvProto tsKvProtoAttribute4 = getTsKvProto("attribute4", "72", TransportProtos.KeyValueType.LONG_V); - TransportProtos.TsKvProto tsKvProtoAttribute5 = getTsKvProto("attribute5", "{\"someNumber\":41,\"someArray\":[],\"someNestedObject\":{\"key\":\"value\"}}", TransportProtos.KeyValueType.JSON_V); - List tsKvProtoList = new ArrayList<>(); - tsKvProtoList.add(tsKvProtoAttribute1); - tsKvProtoList.add(tsKvProtoAttribute2); - tsKvProtoList.add(tsKvProtoAttribute3); - tsKvProtoList.add(tsKvProtoAttribute4); - tsKvProtoList.add(tsKvProtoAttribute5); - TransportProtos.AttributeUpdateNotificationMsg expectedCurrentStateNotificationMsg = expectedCurrentStateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList).build(); - TransportProtos.AttributeUpdateNotificationMsg actualCurrentStateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - List expectedSharedUpdatedList = expectedCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List actualSharedUpdatedList = actualCurrentStateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - - assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); - assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); - - } - - protected void validateEmptyCurrentStateAttributesResponse(TestCoapCallback callback) throws InvalidProtocolBufferException { - assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(0, callback.getObserve().intValue()); - } - - protected void validateUpdateAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - List tsKvProtoList = getTsKvProtoList(); - attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); - - TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); - TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - List actualSharedUpdatedList = actualAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - List expectedSharedUpdatedList = expectedAttributeUpdateNotificationMsg.getSharedUpdatedList().stream().map(TransportProtos.TsKvProto::getKv).collect(Collectors.toList()); - - assertEquals(expectedSharedUpdatedList.size(), actualSharedUpdatedList.size()); - assertTrue(actualSharedUpdatedList.containsAll(expectedSharedUpdatedList)); - - } - - protected void validateDeleteAttributesResponse(TestCoapCallback callback, int expectedObserveCnt) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveCnt, callback.getObserve().intValue()); - TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); - attributeUpdateNotificationMsgBuilder.addSharedDeleted("attribute5"); - - TransportProtos.AttributeUpdateNotificationMsg expectedAttributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); - TransportProtos.AttributeUpdateNotificationMsg actualAttributeUpdateNotificationMsg = TransportProtos.AttributeUpdateNotificationMsg.parseFrom(callback.getPayloadBytes()); - - assertEquals(expectedAttributeUpdateNotificationMsg.getSharedDeletedList().size(), actualAttributeUpdateNotificationMsg.getSharedDeletedList().size()); - assertEquals("attribute5", actualAttributeUpdateNotificationMsg.getSharedDeletedList().get(0)); - - } - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java index de9c024bb8..e2be2280b2 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimDeviceTest.java @@ -16,10 +16,8 @@ package org.thingsboard.server.transport.coap.claim; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; @@ -34,6 +32,7 @@ import org.thingsboard.server.dao.device.claim.ClaimResponse; import org.thingsboard.server.dao.device.claim.ClaimResult; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; import java.io.IOException; @@ -96,7 +95,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { log.warn("[testClaimingDevice] Device: {}, Transport type: {}", savedDevice.getName(), savedDevice.getType()); - client = getCoapClient(FeatureType.CLAIM); + client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; byte[] failurePayloadBytes; if (emptyPayload) { @@ -109,7 +108,7 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { validateClaimResponse(emptyPayload, client, payloadBytes, failurePayloadBytes); } - protected void validateClaimResponse(boolean emptyPayload, CoapClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception { + protected void validateClaimResponse(boolean emptyPayload, CoapTestClient client, byte[] payloadBytes, byte[] failurePayloadBytes) throws Exception { postClaimRequest(client, failurePayloadBytes); loginUser(customerAdmin.getName(), CUSTOMER_USER_PASSWORD); @@ -145,8 +144,8 @@ public class CoapClaimDeviceTest extends AbstractCoapIntegrationTest { assertEquals(claimResponse, ClaimResponse.CLAIMED); } - private void postClaimRequest(CoapClient client, byte[] payload) throws IOException, ConnectorException { - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + private void postClaimRequest(CoapTestClient client, byte[] payload) throws IOException, ConnectorException { + CoapResponse coapResponse = client.postMethod(payload); assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java index 812ea7392b..0096827af7 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/claim/CoapClaimProtoDeviceTest.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.gen.transport.TransportApiProtos; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; @Slf4j @@ -56,7 +57,7 @@ public class CoapClaimProtoDeviceTest extends CoapClaimDeviceTest { @Override protected void processTestClaimingDevice(boolean emptyPayload) throws Exception { - client = getCoapClient(FeatureType.CLAIM); + client = new CoapTestClient(accessToken, FeatureType.CLAIM); byte[] payloadBytes; if (emptyPayload) { TransportApiProtos.ClaimDevice claimDevice = getClaimDevice(0, emptyPayload); diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java index a952ef6d63..ce5ca909ba 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionJsonDeviceTest.java @@ -15,16 +15,13 @@ */ package org.thingsboard.server.transport.coap.provision; -import com.google.gson.JsonObject; +import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Assert; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.common.data.CoapDeviceType; @@ -34,14 +31,12 @@ import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.msg.EncryptionUtil; import org.thingsboard.server.common.msg.session.FeatureType; -import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.dao.device.DeviceCredentialsService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.device.provision.ProvisionResponseStatus; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; - @Slf4j @DaoSqlTest public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { @@ -95,10 +90,11 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .transportPayloadType(TransportPayloadType.JSON) .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); - Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("errorMsg")); + Assert.assertTrue(response.hasNonNull("status")); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText()); } @@ -112,8 +108,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -121,8 +118,8 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } @@ -137,8 +134,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .build(); processBeforeTest(configProperties); String requestCredentials = ",\"credentialsType\": \"ACCESS_TOKEN\",\"token\": \"test_token\""; - byte[] result = createCoapClientAndPublish(requestCredentials).getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials)); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -146,10 +144,10 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "ACCESS_TOKEN"); Assert.assertEquals(deviceCredentials.getCredentialsId(), "test_token"); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } @@ -164,8 +162,9 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .build(); processBeforeTest(configProperties); String requestCredentials = ",\"credentialsType\": \"X509_CERTIFICATE\",\"hash\": \"testHash\""; - byte[] result = createCoapClientAndPublish(requestCredentials).getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish(requestCredentials)); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -173,7 +172,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); Assert.assertEquals(deviceCredentials.getCredentialsType().name(), "X509_CERTIFICATE"); String cert = EncryptionUtil.certTrimNewLines(deviceCredentials.getCredentialsValue()); @@ -182,7 +181,7 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { Assert.assertEquals(deviceCredentials.getCredentialsId(), sha3Hash); Assert.assertEquals(deviceCredentials.getCredentialsValue(), "testHash"); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } private void processTestProvisioningCheckPreProvisionedDevice() throws Exception { @@ -195,13 +194,14 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("credentialsType")); + Assert.assertTrue(response.hasNonNull("status")); DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").getAsString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.get("credentialsType").asText()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.get("status").asText()); } private void processTestProvisioningWithBadKeyDevice() throws Exception { @@ -214,25 +214,21 @@ public class CoapProvisionJsonDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - byte[] result = createCoapClientAndPublish().getPayload(); - JsonObject response = JsonUtils.parse(new String(result)).getAsJsonObject(); - Assert.assertEquals("Provision data was not found!", response.get("errorMsg").getAsString()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").getAsString()); + JsonNode response = JacksonUtil.fromBytes(createCoapClientAndPublish()); + Assert.assertTrue(response.hasNonNull("errorMsg")); + Assert.assertTrue(response.hasNonNull("status")); + Assert.assertEquals("Provision data was not found!", response.get("errorMsg").asText()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.get("status").asText()); } - private CoapResponse createCoapClientAndPublish() throws Exception { + private byte[] createCoapClientAndPublish() throws Exception { return createCoapClientAndPublish(""); } - private CoapResponse createCoapClientAndPublish(String deviceCredentials) throws Exception { + private byte[] createCoapClientAndPublish(String deviceCredentials) throws Exception { String provisionRequestMsg = createTestProvisionMessage(deviceCredentials); - client = getCoapClient(FeatureType.PROVISION); - return postProvision(client, provisionRequestMsg.getBytes()); - } - - - private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException { - return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + client = new CoapTestClient(accessToken, FeatureType.PROVISION); + return client.postMethod(provisionRequestMsg.getBytes()).getPayload(); } private String createTestProvisionMessage(String deviceCredentials) { diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java index a2e270796e..a5253b8409 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/provision/CoapProvisionProtoDeviceTest.java @@ -16,10 +16,7 @@ package org.thingsboard.server.transport.coap.provision; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Assert; import org.junit.Test; @@ -44,10 +41,9 @@ import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceReque import org.thingsboard.server.gen.transport.TransportProtos.ProvisionDeviceResponseMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceTokenRequestMsg; import org.thingsboard.server.gen.transport.TransportProtos.ValidateDeviceX509CertRequestMsg; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; - @Slf4j @DaoSqlTest public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { @@ -101,9 +97,9 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .transportPayloadType(TransportPayloadType.PROTOBUF) .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg result = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); Assert.assertNotNull(result); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().toString()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), result.getStatus().name()); } private void processTestProvisioningCreateNewDeviceWithoutCredentials() throws Exception { @@ -116,7 +112,7 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -124,8 +120,8 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, createdDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name()); } private void processTestProvisioningCreateNewDeviceWithAccessToken() throws Exception { @@ -138,9 +134,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()).build(); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder() + .setValidateDeviceTokenRequestMsg(ValidateDeviceTokenRequestMsg.newBuilder().setToken("test_token").build()) + .build(); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials)).getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom( + createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.ACCESS_TOKEN, requestCredentials))); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -164,9 +163,13 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder().setValidateDeviceX509CertRequestMsg(ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()).build(); + CredentialsDataProto requestCredentials = CredentialsDataProto.newBuilder() + .setValidateDeviceX509CertRequestMsg( + ValidateDeviceX509CertRequestMsg.newBuilder().setHash("testHash").build()) + .build(); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials)).getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom( + createCoapClientAndPublish(createTestsProvisionMessage(CredentialsType.X509_CERTIFICATE, requestCredentials))); Device createdDevice = deviceService.findDeviceByTenantIdAndName(tenantId, "Test Provision device"); @@ -196,12 +199,12 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); DeviceCredentials deviceCredentials = deviceCredentialsService.findDeviceCredentialsByDeviceId(tenantId, savedDevice.getId()); - Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().toString()); - Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().toString()); + Assert.assertEquals(deviceCredentials.getCredentialsType().name(), response.getCredentialsType().name()); + Assert.assertEquals(ProvisionResponseStatus.SUCCESS.name(), response.getStatus().name()); } private void processTestProvisioningWithBadKeyDevice() throws Exception { @@ -214,24 +217,19 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .provisionSecret("testProvisionSecret") .build(); processBeforeTest(configProperties); - ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish().getPayload()); - Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().toString()); - } - - private CoapResponse createCoapClientAndPublish() throws Exception { - byte[] provisionRequestMsg = createTestProvisionMessage(); - CoapResponse coapResponse = createCoapClientAndPublish(provisionRequestMsg); - Assert.assertNotNull("COAP response", coapResponse); - return coapResponse; + ProvisionDeviceResponseMsg response = ProvisionDeviceResponseMsg.parseFrom(createCoapClientAndPublish()); + Assert.assertEquals(ProvisionResponseStatus.NOT_FOUND.name(), response.getStatus().name()); } - private CoapResponse createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception { - client = getCoapClient(FeatureType.PROVISION); - return postProvision(client, provisionRequestMsg); + private byte[] createCoapClientAndPublish() throws Exception { + return createCoapClientAndPublish(createTestProvisionMessage()); } - private CoapResponse postProvision(CoapClient client, byte[] payload) throws IOException, ConnectorException { - return client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); + private byte[] createCoapClientAndPublish(byte[] provisionRequestMsg) throws Exception { + client = new CoapTestClient(accessToken, FeatureType.PROVISION); + CoapResponse coapResponse = client.postMethod(provisionRequestMsg); + Assert.assertNotNull("COAP response", coapResponse); + return coapResponse.getPayload(); } private byte[] createTestsProvisionMessage(CredentialsType credentialsType, CredentialsDataProto credentialsData) throws Exception { @@ -247,7 +245,6 @@ public class CoapProvisionProtoDeviceTest extends AbstractCoapIntegrationTest { .toByteArray(); } - private byte[] createTestProvisionMessage() throws Exception { return createTestsProvisionMessage(null, null); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java index 5262b55b97..beb4d3747c 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/AbstractCoapServerSideRpcIntegrationTest.java @@ -16,18 +16,29 @@ package org.thingsboard.server.transport.coap.rpc; import com.fasterxml.jackson.databind.JsonNode; +import com.github.os72.protobuf.dynamic.DynamicSchema; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapHandler; import org.eclipse.californium.core.CoapObserveRelation; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.core.coap.Request; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; +import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; +import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; +import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestCallback; +import org.thingsboard.server.transport.coap.CoapTestClient; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -41,76 +52,74 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractCoapIntegrationTest { + public static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" + + "package rpc;\n" + + "\n" + + "message RpcRequestMsg {\n" + + " optional string method = 1;\n" + + " optional int32 requestId = 2;\n" + + " Params params = 3;\n" + + "\n" + + " message Params {\n" + + " optional string pin = 1;\n" + + " optional int32 value = 2;\n" + + " }\n" + + "}"; + protected static final String DEVICE_RESPONSE = "{\"value1\":\"A\",\"value2\":\"B\"}"; protected static final Long asyncContextTimeoutToUseRpcPlugin = 10000L; - protected void processOneWayRpcTest() throws Exception { - client = getCoapClient(FeatureType.RPC); - client.useCONs(); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(client, latch, true); + protected void processOneWayRpcTest(boolean protobuf) throws Exception { + client = new CoapTestClient(accessToken, FeatureType.RPC); + CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, true, protobuf); - Request request = Request.newGet().setObserve(); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); - - validateCurrentStateNotification(callback); - - latch = new CountDownLatch(1); + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); + validateCurrentStateNotification(callbackCoap); + CountDownLatch latch = new CountDownLatch(1); String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"23\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String result = doPostAsync("/api/rpc/oneway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateOneWayStateChangedNotification(callback, result); + validateOneWayStateChangedNotification(callbackCoap, result); observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); } - protected void processTwoWayRpcTest(String expectedResponseResult) throws Exception { - client = getCoapClient(FeatureType.RPC); - client.useCONs(); - - CountDownLatch latch = new CountDownLatch(1); - TestCoapCallback callback = new TestCoapCallback(client, latch, false); + protected void processTwoWayRpcTest(String expectedResponseResult, boolean protobuf) throws Exception { + client = new CoapTestClient(accessToken, FeatureType.RPC); + CoapTestCallback callbackCoap = new TestCoapCallbackForRPC(client, 1, false, protobuf); - Request request = Request.newGet().setObserve(); - request.setType(CoAP.Type.CON); - CoapObserveRelation observeRelation = client.observe(request, callback); - - latch.await(3, TimeUnit.SECONDS); + CoapObserveRelation observeRelation = client.getObserveRelation(callbackCoap); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - validateCurrentStateNotification(callback); + validateCurrentStateNotification(callbackCoap); String setGpioRequest = "{\"method\":\"setGpio\",\"params\":{\"pin\": \"26\",\"value\": 1}}"; String deviceId = savedDevice.getId().getId().toString(); String actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); - - validateTwoWayStateChangedNotification(callback, 1, expectedResponseResult, actualResult); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - latch = new CountDownLatch(1); + validateTwoWayStateChangedNotification(callbackCoap, 1, expectedResponseResult, actualResult); + CountDownLatch latch = new CountDownLatch(1); actualResult = doPostAsync("/api/rpc/twoway/" + deviceId, setGpioRequest, String.class, status().isOk()); - latch.await(3, TimeUnit.SECONDS); + callbackCoap.getLatch().await(3, TimeUnit.SECONDS); - validateTwoWayStateChangedNotification(callback, 2, expectedResponseResult, actualResult); + validateTwoWayStateChangedNotification(callbackCoap, 2, expectedResponseResult, actualResult); observeRelation.proactiveCancel(); assertTrue(observeRelation.isCanceled()); } - protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { + protected void processOnLoadResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { JsonNode responseJson = JacksonUtil.fromBytes(response.getPayload()); - client.setURI(getRpcResponseFeatureTokenUrl(accessToken, responseJson.get("id").asInt())); - client.post(new CoapHandler() { + client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, responseJson.get("id").asInt())); + client.postMethod(new CoapHandler() { @Override public void onLoad(CoapResponse response) { log.warn("Command Response Ack: {}, {}", response.getCode(), response.getResponseText()); @@ -124,36 +133,89 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC }, DEVICE_RESPONSE, MediaTypeRegistry.APPLICATION_JSON); } - protected String getRpcResponseFeatureTokenUrl(String token, int requestId) { - return COAP_BASE_URL + token + "/" + FeatureType.RPC.name().toLowerCase() + "/" + requestId; + protected void processOnLoadProtoResponse(CoapResponse response, CoapTestClient client, Integer observe, CountDownLatch latch) { + ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); + ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); + DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); + + byte[] requestPayload = response.getPayload(); + DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); + Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType(); + try { + DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); + Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); + int requestId = (int) dynamicMessage.getField(requestIdDescriptor); + ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); + DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); + DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); + Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); + DynamicMessage rpcResponseMsg = rpcResponseBuilder + .setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE) + .build(); + client.setURI(CoapTestClient.getFeatureTokenUrl(accessToken, FeatureType.RPC, requestId)); + client.postMethod(new CoapHandler() { + @Override + public void onLoad(CoapResponse response) { + log.warn("Command Response Ack: {}", response.getCode()); + latch.countDown(); + } + + @Override + public void onError() { + log.warn("Command Response Ack Error, No connect"); + } + }, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON); + } catch (InvalidProtocolBufferException e) { + log.warn("Command Response Ack Error, Invalid response received: ", e); + } } - protected class TestCoapCallback implements CoapHandler { + private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() { + DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); + assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); + CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; + CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration(); + assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); + DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; + TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); + assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); + return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + } - private final CoapClient client; - private final CountDownLatch latch; - private final boolean isOneWayRpc; + private void validateCurrentStateNotification(CoapTestCallback callback) { + assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID); + assertEquals(0, callback.getObserve().intValue()); + } - private Integer observe; - private byte[] payloadBytes; - private CoAP.ResponseCode responseCode; + private void validateOneWayStateChangedNotification(CoapTestCallback callback, String result) { + assertTrue(StringUtils.isEmpty(result)); + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(1, callback.getObserve().intValue()); + } - public Integer getObserve() { - return observe; - } + private void validateTwoWayStateChangedNotification(CoapTestCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) { + assertEquals(expectedResult, actualResult); + assertNotNull(callback.getPayloadBytes()); + assertNotNull(callback.getObserve()); + assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); + assertEquals(expectedObserveNumber, callback.getObserve().intValue()); + } - public byte[] getPayloadBytes() { - return payloadBytes; - } + protected class TestCoapCallbackForRPC extends CoapTestCallback { - public CoAP.ResponseCode getResponseCode() { - return responseCode; - } + private final CoapTestClient client; + private final boolean isOneWayRpc; + private final boolean protobuf; - TestCoapCallback(CoapClient client, CountDownLatch latch, boolean isOneWayRpc) { + TestCoapCallbackForRPC(CoapTestClient client, int subscribeCount, boolean isOneWayRpc, boolean protobuf) { + super(subscribeCount); this.client = client; - this.latch = latch; this.isOneWayRpc = isOneWayRpc; + this.protobuf = protobuf; } @Override @@ -163,7 +225,11 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC observe = response.getOptions().getObserve(); if (observe != null) { if (!isOneWayRpc && observe > 0) { - processOnLoadResponse(response, client, observe, latch); + if (!protobuf){ + processOnLoadResponse(response, client, observe, latch); + } else { + processOnLoadProtoResponse(response, client, observe, latch); + } } else { latch.countDown(); } @@ -174,31 +240,5 @@ public abstract class AbstractCoapServerSideRpcIntegrationTest extends AbstractC public void onError() { log.warn("Command Response Ack Error, No connect"); } - - } - - private void validateCurrentStateNotification(TestCoapCallback callback) { - assertArrayEquals(EMPTY_PAYLOAD, callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(callback.getResponseCode(), CoAP.ResponseCode.VALID); - assertEquals(0, callback.getObserve().intValue()); } - - private void validateOneWayStateChangedNotification(TestCoapCallback callback, String result) { - assertTrue(StringUtils.isEmpty(result)); - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(1, callback.getObserve().intValue()); - } - - private void validateTwoWayStateChangedNotification(TestCoapCallback callback, int expectedObserveNumber, String expectedResult, String actualResult) { - assertEquals(expectedResult, actualResult); - assertNotNull(callback.getPayloadBytes()); - assertNotNull(callback.getObserve()); - assertEquals(CoAP.ResponseCode.CONTENT, callback.getResponseCode()); - assertEquals(expectedObserveNumber, callback.getObserve().intValue()); - } - - } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java index cd0e9268b4..d2ef37f795 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcDefaultIntegrationTest.java @@ -21,9 +21,6 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; -import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; @@ -87,12 +84,12 @@ public class CoapServerSideRpcDefaultIntegrationTest extends AbstractCoapServerS @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(false); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}"); + processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java index 6dde4b919b..96ab02325a 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcJsonIntegrationTest.java @@ -45,12 +45,12 @@ public class CoapServerSideRpcJsonIntegrationTest extends AbstractCoapServerSide @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(false); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}"); + processTwoWayRpcTest("{\"value1\":\"A\",\"value2\":\"B\"}", false); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java index ae1c89d765..93b5c6c6fb 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/rpc/CoapServerSideRpcProtoIntegrationTest.java @@ -15,52 +15,19 @@ */ package org.thingsboard.server.transport.coap.rpc; -import com.github.os72.protobuf.dynamic.DynamicSchema; -import com.google.protobuf.Descriptors; -import com.google.protobuf.DynamicMessage; -import com.google.protobuf.InvalidProtocolBufferException; -import com.squareup.wire.schema.internal.parser.ProtoFileElement; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; -import org.eclipse.californium.core.CoapHandler; -import org.eclipse.californium.core.CoapResponse; -import org.eclipse.californium.core.coap.MediaTypeRegistry; import org.junit.After; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; import org.thingsboard.server.common.data.TransportPayloadType; -import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DefaultCoapDeviceTypeConfiguration; -import org.thingsboard.server.common.data.device.profile.DeviceProfileTransportConfiguration; -import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; -import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.util.concurrent.CountDownLatch; - -import static org.junit.Assert.assertTrue; - @Slf4j @DaoSqlTest public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSideRpcIntegrationTest { - private static final String RPC_REQUEST_PROTO_SCHEMA = "syntax =\"proto3\";\n" + - "package rpc;\n" + - "\n" + - "message RpcRequestMsg {\n" + - " optional string method = 1;\n" + - " optional int32 requestId = 2;\n" + - " Params params = 3;\n" + - "\n" + - " message Params {\n" + - " optional string pin = 1;\n" + - " optional int32 value = 2;\n" + - " }\n" + - "}"; - @Before public void beforeTest() throws Exception { CoapTestConfigProperties configProperties = CoapTestConfigProperties.builder() @@ -79,61 +46,11 @@ public class CoapServerSideRpcProtoIntegrationTest extends AbstractCoapServerSid @Test public void testServerCoapOneWayRpc() throws Exception { - processOneWayRpcTest(); + processOneWayRpcTest(true); } @Test public void testServerCoapTwoWayRpc() throws Exception { - processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}"); - } - - @Override - protected void processOnLoadResponse(CoapResponse response, CoapClient client, Integer observe, CountDownLatch latch) { - ProtoTransportPayloadConfiguration protoTransportPayloadConfiguration = getProtoTransportPayloadConfiguration(); - ProtoFileElement rpcRequestProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(RPC_REQUEST_PROTO_SCHEMA); - DynamicSchema rpcRequestProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcRequestProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_REQUEST_PROTO_SCHEMA); - - byte[] requestPayload = response.getPayload(); - DynamicMessage.Builder rpcRequestMsg = rpcRequestProtoSchema.newMessageBuilder("RpcRequestMsg"); - Descriptors.Descriptor rpcRequestMsgDescriptor = rpcRequestMsg.getDescriptorForType(); - try { - DynamicMessage dynamicMessage = DynamicMessage.parseFrom(rpcRequestMsgDescriptor, requestPayload); - Descriptors.FieldDescriptor requestIdDescriptor = rpcRequestMsgDescriptor.findFieldByName("requestId"); - int requestId = (int) dynamicMessage.getField(requestIdDescriptor); - ProtoFileElement rpcResponseProtoSchemaFile = protoTransportPayloadConfiguration.getTransportProtoSchema(DEVICE_RPC_RESPONSE_PROTO_SCHEMA); - DynamicSchema rpcResponseProtoSchema = protoTransportPayloadConfiguration.getDynamicSchema(rpcResponseProtoSchemaFile, ProtoTransportPayloadConfiguration.RPC_RESPONSE_PROTO_SCHEMA); - DynamicMessage.Builder rpcResponseBuilder = rpcResponseProtoSchema.newMessageBuilder("RpcResponseMsg"); - Descriptors.Descriptor rpcResponseMsgDescriptor = rpcResponseBuilder.getDescriptorForType(); - DynamicMessage rpcResponseMsg = rpcResponseBuilder - .setField(rpcResponseMsgDescriptor.findFieldByName("payload"), DEVICE_RESPONSE) - .build(); - client.setURI(getRpcResponseFeatureTokenUrl(accessToken, requestId)); - client.post(new CoapHandler() { - @Override - public void onLoad(CoapResponse response) { - log.warn("Command Response Ack: {}", response.getCode()); - latch.countDown(); - } - - @Override - public void onError() { - log.warn("Command Response Ack Error, No connect"); - } - }, rpcResponseMsg.toByteArray(), MediaTypeRegistry.APPLICATION_JSON); - } catch (InvalidProtocolBufferException e) { - log.warn("Command Response Ack Error, Invalid response received: ", e); - } - } - - private ProtoTransportPayloadConfiguration getProtoTransportPayloadConfiguration() { - DeviceProfileTransportConfiguration transportConfiguration = deviceProfile.getProfileData().getTransportConfiguration(); - assertTrue(transportConfiguration instanceof CoapDeviceProfileTransportConfiguration); - CoapDeviceProfileTransportConfiguration coapDeviceProfileTransportConfiguration = (CoapDeviceProfileTransportConfiguration) transportConfiguration; - CoapDeviceTypeConfiguration coapDeviceTypeConfiguration = coapDeviceProfileTransportConfiguration.getCoapDeviceTypeConfiguration(); - assertTrue(coapDeviceTypeConfiguration instanceof DefaultCoapDeviceTypeConfiguration); - DefaultCoapDeviceTypeConfiguration defaultCoapDeviceTypeConfiguration = (DefaultCoapDeviceTypeConfiguration) coapDeviceTypeConfiguration; - TransportPayloadTypeConfiguration transportPayloadTypeConfiguration = defaultCoapDeviceTypeConfiguration.getTransportPayloadTypeConfiguration(); - assertTrue(transportPayloadTypeConfiguration instanceof ProtoTransportPayloadConfiguration); - return (ProtoTransportPayloadConfiguration) transportPayloadTypeConfiguration; + processTwoWayRpcTest("{\"payload\":\"{\\\"value1\\\":\\\"A\\\",\\\"value2\\\":\\\"B\\\"}\"}", true); } } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java index ee566be2bb..057e4366de 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/attributes/CoapAttributesIntegrationTest.java @@ -17,11 +17,8 @@ package org.thingsboard.server.transport.coap.telemetry.attributes; import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -29,9 +26,9 @@ import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.msg.session.FeatureType; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.LinkedHashMap; @@ -74,30 +71,17 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { } protected void processAttributesTest(List expectedKeys, byte[] payload, boolean presenceFieldsTest) throws Exception { - client = getCoapClient(FeatureType.ATTRIBUTES); - postAttributes(client, payload); + client = new CoapTestClient(accessToken, FeatureType.ATTRIBUTES); + CoapResponse coapResponse = client.postMethod(payload); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); DeviceId deviceId = savedDevice.getId(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; - - List actualKeys = null; - while (start <= end) { - actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); - if (actualKeys.size() == expectedKeys.size()) { - break; - } - Thread.sleep(100); - start += 100; - } + List actualKeys = getActualKeysList(deviceId, expectedKeys); assertNotNull(actualKeys); Set actualKeySet = new HashSet<>(actualKeys); - Set expectedKeySet = new HashSet<>(expectedKeys); - assertEquals(expectedKeySet, actualKeySet); String getAttributesValuesUrl = getAttributesValuesUrl(deviceId, actualKeySet); @@ -111,14 +95,6 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { doDelete(deleteAttributesUrl); } - private void postAttributes(CoapClient client, byte[] payload) throws IOException, ConnectorException { - if (payload == null) { - payload = PAYLOAD_VALUES_STR.getBytes(); - } - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - @SuppressWarnings({"unchecked", "rawtypes"}) protected void assertAttributesValues(List> deviceValues, Set keySet) { for (Map map : deviceValues) { @@ -171,6 +147,22 @@ public class CoapAttributesIntegrationTest extends AbstractCoapIntegrationTest { } } + private List getActualKeysList(DeviceId deviceId, List expectedKeys) throws Exception { + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; + + List actualKeys = null; + while (start <= end) { + actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/attributes/CLIENT_SCOPE", new TypeReference<>() {}); + if (actualKeys.size() == expectedKeys.size()) { + break; + } + Thread.sleep(100); + start += 100; + } + return actualKeys; + } + private String getAttributesValuesUrl(DeviceId deviceId, Set actualKeySet) { return "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/attributes/CLIENT_SCOPE?keys=" + String.join(",", actualKeySet); } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java index 0c7ab4d1c5..a1fba145a9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesIntegrationTest.java @@ -17,19 +17,17 @@ package org.thingsboard.server.transport.coap.telemetry.timeseries; import com.fasterxml.jackson.core.type.TypeReference; import lombok.extern.slf4j.Slf4j; -import org.eclipse.californium.core.CoapClient; import org.eclipse.californium.core.CoapResponse; import org.eclipse.californium.core.coap.CoAP; -import org.eclipse.californium.core.coap.MediaTypeRegistry; -import org.eclipse.californium.elements.exception.ConnectorException; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.msg.session.FeatureType; import org.thingsboard.server.transport.coap.AbstractCoapIntegrationTest; +import org.thingsboard.server.transport.coap.CoapTestClient; import org.thingsboard.server.transport.coap.CoapTestConfigProperties; -import java.io.IOException; import java.util.Arrays; import java.util.HashSet; import java.util.List; @@ -75,28 +73,16 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } protected void processTestPostTelemetry(byte[] payloadBytes, List expectedKeys, boolean withTs, boolean presenceFieldsTest) throws Exception { - client = getCoapClient(FeatureType.TELEMETRY); - postTelemetry(client, payloadBytes); - - String deviceId = savedDevice.getId().getId().toString(); - - long start = System.currentTimeMillis(); - long end = System.currentTimeMillis() + 5000; + client = new CoapTestClient(accessToken, FeatureType.TELEMETRY); + CoapResponse coapResponse = client.postMethod(payloadBytes); + assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - List actualKeys = null; - while (start <= end) { - actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {}); - if (actualKeys.size() == expectedKeys.size()) { - break; - } - Thread.sleep(100); - start += 100; - } + DeviceId deviceId = savedDevice.getId(); + List actualKeys = getActualKeysList(deviceId, expectedKeys); assertNotNull(actualKeys); Set actualKeySet = new HashSet<>(actualKeys); Set expectedKeySet = new HashSet<>(expectedKeys); - assertEquals(expectedKeySet, actualKeySet); String getTelemetryValuesUrl; @@ -105,8 +91,8 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } else { getTelemetryValuesUrl = "/api/plugins/telemetry/DEVICE/" + deviceId + "/values/timeseries?keys=" + String.join(",", actualKeySet); } - start = System.currentTimeMillis(); - end = System.currentTimeMillis() + 5000; + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; Map>> values = null; while (start <= end) { values = doGetAsyncTyped(getTelemetryValuesUrl, new TypeReference<>() {}); @@ -144,11 +130,6 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } } - private void postTelemetry(CoapClient client, byte[] payload) throws IOException, ConnectorException { - CoapResponse coapResponse = client.setTimeout((long) 60000).post(payload, MediaTypeRegistry.APPLICATION_JSON); - assertEquals(CoAP.ResponseCode.CREATED, coapResponse.getCode()); - } - private void assertTs(Map>> deviceValues, List expectedKeys, int ts, int arrayIndex) { assertEquals(ts, deviceValues.get(expectedKeys.get(0)).get(arrayIndex).get("ts")); assertEquals(ts, deviceValues.get(expectedKeys.get(1)).get(arrayIndex).get("ts")); @@ -207,4 +188,20 @@ public abstract class AbstractCoapTimeseriesIntegrationTest extends AbstractCoap } } + private List getActualKeysList(DeviceId deviceId, List expectedKeys) throws Exception { + long start = System.currentTimeMillis(); + long end = System.currentTimeMillis() + 5000; + + List actualKeys = null; + while (start <= end) { + actualKeys = doGetAsyncTyped("/api/plugins/telemetry/DEVICE/" + deviceId + "/keys/timeseries", new TypeReference<>() {}); + if (actualKeys.size() == expectedKeys.size()) { + break; + } + Thread.sleep(100); + start += 100; + } + return actualKeys; + } + } diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java index 93e132663e..fb6d8aa0da 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/telemetry/timeseries/AbstractCoapTimeseriesProtoIntegrationTest.java @@ -23,7 +23,6 @@ import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; import org.thingsboard.server.common.data.CoapDeviceType; -import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.TransportPayloadType; import org.thingsboard.server.common.data.device.profile.CoapDeviceProfileTransportConfiguration; import org.thingsboard.server.common.data.device.profile.CoapDeviceTypeConfiguration; diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java index 37207798fb..ce16bcc00f 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/RedisTbTransactionalCache.java @@ -48,7 +48,7 @@ public abstract class RedisTbTransactionalCache keySerializer = StringRedisSerializer.UTF_8; - private final RedisSerializer valueSerializer; + private final TbRedisSerializer valueSerializer; private final Expiration evictExpiration; private final Expiration cacheTtl; @@ -56,7 +56,7 @@ public abstract class RedisTbTransactionalCache valueSerializer) { + TbRedisSerializer valueSerializer) { this.cacheName = cacheName; this.connectionFactory = (JedisConnectionFactory) connectionFactory; this.valueSerializer = valueSerializer; @@ -79,7 +79,7 @@ public abstract class RedisTbTransactionalCache implements TbCacheValueWrapper { diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbFSTRedisSerializer.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbFSTRedisSerializer.java new file mode 100644 index 0000000000..ff1616d532 --- /dev/null +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbFSTRedisSerializer.java @@ -0,0 +1,32 @@ +/** + * Copyright ยฉ 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.cache; + +import org.springframework.data.redis.serializer.SerializationException; +import org.thingsboard.server.common.data.FSTUtils; + +public class TbFSTRedisSerializer implements TbRedisSerializer { + + @Override + public byte[] serialize(V value) throws SerializationException { + return FSTUtils.encode(value); + } + + @Override + public V deserialize(K key, byte[] bytes) throws SerializationException { + return FSTUtils.decode(bytes); + } +} diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/TbRedisSerializer.java b/common/cache/src/main/java/org/thingsboard/server/cache/TbRedisSerializer.java index 574038ea91..cb8a3cfa6a 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/TbRedisSerializer.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/TbRedisSerializer.java @@ -15,20 +15,15 @@ */ package org.thingsboard.server.cache; -import org.springframework.data.redis.serializer.RedisSerializer; import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.lang.Nullable; -public class TbRedisSerializer implements RedisSerializer { +public interface TbRedisSerializer { - private final RedisSerializer java = RedisSerializer.java(); + @Nullable + byte[] serialize(@Nullable T t) throws SerializationException; - @Override - public byte[] serialize(T t) throws SerializationException { - return java.serialize(t); - } + @Nullable + T deserialize(K key, @Nullable byte[] bytes) throws SerializationException; - @Override - public T deserialize(byte[] bytes) throws SerializationException { - return (T) java.deserialize(bytes); - } } diff --git a/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceRedisCache.java b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceRedisCache.java index 99577f0039..805e70263a 100644 --- a/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceRedisCache.java +++ b/common/cache/src/main/java/org/thingsboard/server/cache/device/DeviceRedisCache.java @@ -21,7 +21,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.Device; @@ -30,6 +30,6 @@ import org.thingsboard.server.common.data.Device; public class DeviceRedisCache extends RedisTbTransactionalCache { public DeviceRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.DEVICE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.DEVICE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/common/cluster-api/src/main/proto/queue.proto b/common/cluster-api/src/main/proto/queue.proto index 45cbda66d3..324df48c2a 100644 --- a/common/cluster-api/src/main/proto/queue.proto +++ b/common/cluster-api/src/main/proto/queue.proto @@ -85,6 +85,17 @@ message KeyValueProto { string json_v = 7; } +message AttributeValueProto { + int64 lastUpdateTs = 1; + KeyValueType type = 2; + bool has_v = 3; + bool bool_v = 4; + int64 long_v = 5; + double double_v = 6; + string string_v = 7; + string json_v = 8; +} + message TsKvProto { int64 ts = 1; KeyValueProto kv = 2; @@ -715,7 +726,10 @@ message PrepareMsg { message AddMsg { string relativePath = 1; - string entityDataJson = 2; + string entityDataJsonChunk = 2; + string chunkedMsgId = 3; + int32 chunkIndex = 4; + int32 chunksCount = 5; } message DeleteMsg { @@ -755,9 +769,8 @@ message ListVersionsResponseMsg { } message ListEntitiesRequestMsg { - string branchName = 1; - string versionId = 2; - string entityType = 3; + string versionId = 1; + string entityType = 2; } message VersionedEntityInfoProto { @@ -791,6 +804,9 @@ message EntityContentRequestMsg { message EntityContentResponseMsg { string data = 1; + string chunkedMsgId = 2; + int32 chunkIndex = 3; + int32 chunksCount = 4; } message EntitiesContentRequestMsg { @@ -801,7 +817,8 @@ message EntitiesContentRequestMsg { } message EntitiesContentResponseMsg { - repeated string data = 1; + EntityContentResponseMsg item = 1; + int32 itemsCount = 2; } message VersionsDiffRequestMsg { @@ -823,15 +840,6 @@ message EntityVersionsDiff { string rawDiff = 6; } -message ContentsDiffRequestMsg { - string content1 = 1; - string content2 = 2; -} - -message ContentsDiffResponseMsg { - string diff = 1; -} - message GenericRepositoryRequestMsg {} message GenericRepositoryResponseMsg {} @@ -853,7 +861,6 @@ message ToVersionControlServiceMsg { EntityContentRequestMsg entityContentRequest = 14; EntitiesContentRequestMsg entitiesContentRequest = 15; VersionsDiffRequestMsg versionsDiffRequest = 16; - ContentsDiffRequestMsg contentsDiffRequest = 17; } message VersionControlResponseMsg { @@ -868,7 +875,6 @@ message VersionControlResponseMsg { EntityContentResponseMsg entityContentResponse = 9; EntitiesContentResponseMsg entitiesContentResponse = 10; VersionsDiffResponseMsg versionsDiffResponse = 11; - ContentsDiffResponseMsg contentsDiffResponse = 12; } /** diff --git a/common/data/pom.xml b/common/data/pom.xml index 6c145553c0..539dd0d9e3 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -104,6 +104,10 @@ io.swagger swagger-annotations + + de.ruedigermoeller + fst + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/FSTUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/FSTUtils.java new file mode 100644 index 0000000000..f9da8f44fb --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/FSTUtils.java @@ -0,0 +1,35 @@ +/** + * Copyright ยฉ 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data; + +import lombok.extern.slf4j.Slf4j; +import org.nustaq.serialization.FSTConfiguration; + +@Slf4j +public class FSTUtils { + + public static final FSTConfiguration CONFIG = FSTConfiguration.createDefaultConfiguration(); + + @SuppressWarnings("unchecked") + public static T decode(byte[] byteArray) { + return byteArray != null && byteArray.length > 0 ? (T) CONFIG.asObject(byteArray) : null; + } + + public static byte[] encode(T msq) { + return CONFIG.asByteArray(msq); + } + +} diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index ff3d3c0fd4..f76757f522 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.common.data; +import com.google.common.base.Splitter; + import static org.apache.commons.lang3.StringUtils.repeat; public class StringUtils { @@ -92,4 +94,8 @@ public class StringUtils { return input.substring(0, startIndexInclusive) + obfuscatedPart + input.substring(endIndexExclusive); } + public static Iterable split(String value, int maxPartSize) { + return Splitter.fixedLength(maxPartSize).split(value); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java index 800583b0b8..dae2b99106 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityDataDiff.java @@ -24,5 +24,4 @@ import org.thingsboard.server.common.data.sync.ie.EntityExportData; public class EntityDataDiff { private EntityExportData currentVersion; private EntityExportData otherVersion; - private String rawDiff; } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java index bc3475c89e..cd3dc5e0e2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityLoadError.java @@ -20,12 +20,15 @@ import lombok.Builder; import lombok.Data; import org.thingsboard.server.common.data.id.EntityId; +import java.io.Serializable; import java.util.List; @Data @Builder @JsonInclude(JsonInclude.Include.NON_NULL) -public class EntityLoadError { +public class EntityLoadError implements Serializable { + + private static final long serialVersionUID = 7538450180582109391L; private String type; private EntityId source; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java index 84a28d0770..06df7ebb83 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/EntityTypeLoadResult.java @@ -21,11 +21,15 @@ import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.EntityType; +import java.io.Serializable; + @Data @AllArgsConstructor @NoArgsConstructor @Builder -public class EntityTypeLoadResult { +public class EntityTypeLoadResult implements Serializable { + private static final long serialVersionUID = -8428039809651395241L; + private EntityType entityType; private int created; private int updated; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java index fd278cde1f..ce8c91f4ae 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/vc/VersionedEntityInfo.java @@ -23,7 +23,6 @@ import org.thingsboard.server.common.data.id.EntityId; @NoArgsConstructor public class VersionedEntityInfo { private EntityId externalId; - // etc.. public VersionedEntityInfo(EntityId externalId) { this.externalId = externalId; diff --git a/common/queue/pom.xml b/common/queue/pom.xml index a625ec6e93..e2197d566f 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -120,10 +120,6 @@ com.google.protobuf protobuf-java-util - - de.ruedigermoeller - fst - org.apache.curator curator-recipes diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java index f1c55973db..93d37560e2 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/util/ProtoWithFSTService.java @@ -18,7 +18,7 @@ package org.thingsboard.server.queue.util; import lombok.extern.slf4j.Slf4j; import org.nustaq.serialization.FSTConfiguration; import org.springframework.stereotype.Service; -import org.thingsboard.server.queue.util.DataDecodingEncodingService; +import org.thingsboard.server.common.data.FSTUtils; import java.util.Optional; @@ -26,23 +26,23 @@ import java.util.Optional; @Service public class ProtoWithFSTService implements DataDecodingEncodingService { - private final FSTConfiguration config = FSTConfiguration.createDefaultConfiguration(); + public static final FSTConfiguration CONFIG = FSTConfiguration.createDefaultConfiguration(); @Override public Optional decode(byte[] byteArray) { try { - @SuppressWarnings("unchecked") - T msg = byteArray != null && byteArray.length > 0 ? (T) config.asObject(byteArray) : null; - return Optional.ofNullable(msg); + return Optional.ofNullable(FSTUtils.decode(byteArray)); } catch (IllegalArgumentException e) { log.error("Error during deserialization message, [{}]", e.getMessage()); return Optional.empty(); } } + @Override public byte[] encode(T msq) { - return config.asByteArray(msq); + return FSTUtils.encode(msq); } + } diff --git a/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java b/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java index 7e39f9a273..adcffb8d21 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java +++ b/common/util/src/main/java/org/thingsboard/common/util/CollectionsUtil.java @@ -39,4 +39,12 @@ public class CollectionsUtil { return isNotEmpty(collection) && collection.contains(element); } + public static int countNonNull(T[] array) { + int count = 0; + for (T t : array) { + if (t != null) count++; + } + return count; + } + } diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java index f74f18ddaa..7d8e81d1c4 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/DefaultClusterVersionControlService.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.service.sync.vc; +import com.google.common.collect.Iterables; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; @@ -28,6 +29,7 @@ import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.CollectionsUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; @@ -89,6 +91,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Function; @@ -123,6 +126,8 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe private long packProcessingTimeout; @Value("${vc.git.io_pool_size:3}") private int ioPoolSize; + @Value("${queue.vc.msg-chunk-size:500000}") + private int msgChunkSize; //We need to manually manage the threads since tasks for particular tenant need to be processed sequentially. private final List ioThreads = new ArrayList<>(); @@ -252,8 +257,6 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe handleEntitiesContentRequest(ctx, msg.getEntitiesContentRequest()); } else if (msg.hasVersionsDiffRequest()) { handleVersionsDiffRequest(ctx, msg.getVersionsDiffRequest()); - } else if (msg.hasContentsDiffRequest()) { - handleContentsDiffRequest(ctx, msg.getContentsDiffRequest()); } } } @@ -270,19 +273,49 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe String path = getRelativePath(entityType, null); var ids = vcService.listEntitiesAtVersion(ctx.getTenantId(), request.getVersionId(), path) .stream().skip(request.getOffset()).limit(request.getLimit()).collect(Collectors.toList()); - var response = EntitiesContentResponseMsg.newBuilder(); - for (VersionedEntityInfo info : ids) { - var data = vcService.getFileContentAtCommit(ctx.getTenantId(), - getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); - response.addData(data); + if (!ids.isEmpty()) { + for (VersionedEntityInfo info : ids) { + var data = vcService.getFileContentAtCommit(ctx.getTenantId(), + getRelativePath(info.getExternalId().getEntityType(), info.getExternalId().getId().toString()), request.getVersionId()); + + Iterable dataChunks = StringUtils.split(data, msgChunkSize); + String chunkedMsgId = UUID.randomUUID().toString(); + int chunksCount = Iterables.size(dataChunks); + AtomicInteger chunkIndex = new AtomicInteger(); + dataChunks.forEach(chunk -> { + EntitiesContentResponseMsg.Builder response = EntitiesContentResponseMsg.newBuilder() + .setItemsCount(ids.size()) + .setItem(EntityContentResponseMsg.newBuilder() + .setData(chunk) + .setChunkedMsgId(chunkedMsgId) + .setChunksCount(chunksCount) + .setChunkIndex(chunkIndex.getAndIncrement()) + .build()); + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); + }); + } + } else { + reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse( + EntitiesContentResponseMsg.newBuilder() + .setItemsCount(0))); } - reply(ctx, Optional.empty(), builder -> builder.setEntitiesContentResponse(response)); } private void handleEntityContentRequest(VersionControlRequestCtx ctx, EntityContentRequestMsg request) throws IOException { String path = getRelativePath(EntityType.valueOf(request.getEntityType()), new UUID(request.getEntityIdMSB(), request.getEntityIdLSB()).toString()); String data = vcService.getFileContentAtCommit(ctx.getTenantId(), path, request.getVersionId()); - reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder().setData(data))); + + Iterable dataChunks = StringUtils.split(data, msgChunkSize); + String chunkedMsgId = UUID.randomUUID().toString(); + int chunksCount = Iterables.size(dataChunks); + + AtomicInteger chunkIndex = new AtomicInteger(); + dataChunks.forEach(chunk -> { + log.trace("[{}] sending chunk {} for 'getEntity'", chunkedMsgId, chunkIndex.get()); + reply(ctx, Optional.empty(), builder -> builder.setEntityContentResponse(EntityContentResponseMsg.newBuilder() + .setData(chunk).setChunkedMsgId(chunkedMsgId).setChunksCount(chunksCount) + .setChunkIndex(chunkIndex.getAndIncrement()))); + }); } private void handleListVersions(VersionControlRequestCtx ctx, ListVersionsRequestMsg request) throws Exception { @@ -359,12 +392,6 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe .addAllDiff(diffList))); } - private void handleContentsDiffRequest(VersionControlRequestCtx ctx, TransportProtos.ContentsDiffRequestMsg request) throws IOException { - String diff = vcService.getContentsDiff(ctx.getTenantId(), request.getContent1(), request.getContent2()); - reply(ctx, builder -> builder.setContentsDiffResponse(TransportProtos.ContentsDiffResponseMsg.newBuilder() - .setDiff(diff))); - } - private void handleCommitRequest(VersionControlRequestCtx ctx, CommitRequestMsg request) throws Exception { var tenantId = ctx.getTenantId(); UUID txId = UUID.fromString(request.getTxId()); @@ -416,7 +443,16 @@ public class DefaultClusterVersionControlService extends TbApplicationEventListe } private void addToCommit(VersionControlRequestCtx ctx, PendingCommit commit, AddMsg addMsg) throws IOException { - vcService.add(commit, addMsg.getRelativePath(), addMsg.getEntityDataJson()); + log.trace("[{}] received chunk {} for 'addToCommit'", addMsg.getChunkedMsgId(), addMsg.getChunkIndex()); + Map chunkedMsgs = commit.getChunkedMsgs(); + String[] msgChunks = chunkedMsgs.computeIfAbsent(addMsg.getChunkedMsgId(), id -> new String[addMsg.getChunksCount()]); + msgChunks[addMsg.getChunkIndex()] = addMsg.getEntityDataJsonChunk(); + if (CollectionsUtil.countNonNull(msgChunks) == msgChunks.length) { + log.trace("[{}] collected all chunks for 'addToCommit'", addMsg.getChunkedMsgId()); + String entityDataJson = String.join("", msgChunks); + chunkedMsgs.remove(addMsg.getChunkedMsgId()); + vcService.add(commit, addMsg.getRelativePath(), entityDataJson); + } } private void doAbortCurrentCommit(TenantId tenantId, PendingCommit current) { diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java index ccd5fc685e..20b1991c9f 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/PendingCommit.java @@ -18,7 +18,9 @@ package org.thingsboard.server.service.sync.vc; import lombok.Data; import org.thingsboard.server.common.data.id.TenantId; +import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; @Data public class PendingCommit { @@ -31,9 +33,10 @@ public class PendingCommit { private String versionName; private String authorName; - private String authorEmail; + private Map chunkedMsgs; + public PendingCommit(TenantId tenantId, String nodeId, UUID txId, String branch, String versionName, String authorName, String authorEmail) { this.tenantId = tenantId; this.nodeId = nodeId; @@ -44,4 +47,12 @@ public class PendingCommit { this.authorEmail = authorEmail; this.workingBranch = txId.toString(); } + + public Map getChunkedMsgs() { + if (chunkedMsgs == null) { + chunkedMsgs = new ConcurrentHashMap<>(); + } + return chunkedMsgs; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetRedisCache.java index 9a8526f3cf..4e16f40c5b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/asset/AssetRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/asset/AssetRedisCache.java @@ -23,13 +23,13 @@ import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("AssetCache") public class AssetRedisCache extends RedisTbTransactionalCache { public AssetRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.ASSET_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java index 80c52105fd..dda00543a0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributeRedisCache.java @@ -15,21 +15,99 @@ */ package org.thingsboard.server.dao.attributes; +import com.google.protobuf.InvalidProtocolBufferException; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.serializer.SerializationException; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; +import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.kv.AttributeKvEntry; -import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.common.data.kv.BaseAttributeKvEntry; +import org.thingsboard.server.common.data.kv.BooleanDataEntry; +import org.thingsboard.server.common.data.kv.DoubleDataEntry; +import org.thingsboard.server.common.data.kv.JsonDataEntry; +import org.thingsboard.server.common.data.kv.KvEntry; +import org.thingsboard.server.common.data.kv.LongDataEntry; +import org.thingsboard.server.common.data.kv.StringDataEntry; +import org.thingsboard.server.gen.transport.TransportProtos.AttributeValueProto; +import org.thingsboard.server.gen.transport.TransportProtos.KeyValueType; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("AttributeCache") public class AttributeRedisCache extends RedisTbTransactionalCache { public AttributeRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ATTRIBUTES_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.ATTRIBUTES_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>() { + @Override + public byte[] serialize(AttributeKvEntry attributeKvEntry) throws SerializationException { + AttributeValueProto.Builder builder = AttributeValueProto.newBuilder() + .setLastUpdateTs(attributeKvEntry.getLastUpdateTs()); + switch (attributeKvEntry.getDataType()) { + case BOOLEAN: + attributeKvEntry.getBooleanValue().ifPresent(builder::setBoolV); + builder.setHasV(attributeKvEntry.getBooleanValue().isPresent()); + builder.setType(KeyValueType.BOOLEAN_V); + break; + case STRING: + attributeKvEntry.getStrValue().ifPresent(builder::setStringV); + builder.setHasV(attributeKvEntry.getStrValue().isPresent()); + builder.setType(KeyValueType.STRING_V); + break; + case DOUBLE: + attributeKvEntry.getDoubleValue().ifPresent(builder::setDoubleV); + builder.setHasV(attributeKvEntry.getDoubleValue().isPresent()); + builder.setType(KeyValueType.DOUBLE_V); + break; + case LONG: + attributeKvEntry.getLongValue().ifPresent(builder::setLongV); + builder.setHasV(attributeKvEntry.getLongValue().isPresent()); + builder.setType(KeyValueType.LONG_V); + break; + case JSON: + attributeKvEntry.getJsonValue().ifPresent(builder::setJsonV); + builder.setHasV(attributeKvEntry.getJsonValue().isPresent()); + builder.setType(KeyValueType.JSON_V); + break; + + } + return builder.build().toByteArray(); + } + + @Override + public AttributeKvEntry deserialize(AttributeCacheKey key, byte[] bytes) throws SerializationException { + try { + AttributeValueProto proto = AttributeValueProto.parseFrom(bytes); + boolean hasValue = proto.getHasV(); + KvEntry entry; + switch (proto.getType()) { + case BOOLEAN_V: + entry = new BooleanDataEntry(key.getKey(), hasValue ? proto.getBoolV() : null); + break; + case LONG_V: + entry = new LongDataEntry(key.getKey(), hasValue ? proto.getLongV() : null); + break; + case DOUBLE_V: + entry = new DoubleDataEntry(key.getKey(), hasValue ? proto.getDoubleV() : null); + break; + case STRING_V: + entry = new StringDataEntry(key.getKey(), hasValue ? proto.getStringV() : null); + break; + case JSON_V: + entry = new JsonDataEntry(key.getKey(), hasValue ? proto.getJsonV() : null); + break; + default: + throw new InvalidProtocolBufferException("Unrecognized type: " + proto.getType() + " !"); + } + return new BaseAttributeKvEntry(proto.getLastUpdateTs(), entry); + } catch (InvalidProtocolBufferException e) { + throw new SerializationException(e.getMessage()); + } + } + }); } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsRedisCache.java index 1d8a9455dc..d9f10a325b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceCredentialsRedisCache.java @@ -20,16 +20,16 @@ import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("DeviceCredentialsCache") public class DeviceCredentialsRedisCache extends RedisTbTransactionalCache { public DeviceCredentialsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.DEVICE_CREDENTIALS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.DEVICE_CREDENTIALS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileRedisCache.java index f3d7ad634f..6c6e3d7c00 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceProfileRedisCache.java @@ -20,16 +20,16 @@ import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("DeviceProfileCache") public class DeviceProfileRedisCache extends RedisTbTransactionalCache { public DeviceProfileRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.DEVICE_PROFILE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.DEVICE_PROFILE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeRedisCache.java index ab0b44a525..88f82b7c27 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/edge/EdgeRedisCache.java @@ -23,13 +23,13 @@ import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("EdgeCache") public class EdgeRedisCache extends RedisTbTransactionalCache { public EdgeRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.EDGE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.EDGE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewRedisCache.java index e2ade492ff..aeb8a93594 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entityview/EntityViewRedisCache.java @@ -22,13 +22,13 @@ import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("EntityViewCache") public class EntityViewRedisCache extends RedisTbTransactionalCache { public EntityViewRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.ENTITY_VIEW_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.ENTITY_VIEW_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageRedisCache.java index 1257932b79..418671f1e2 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageRedisCache.java @@ -20,16 +20,16 @@ import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.OtaPackageInfo; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("OtaPackageCache") public class OtaPackageRedisCache extends RedisTbTransactionalCache { public OtaPackageRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.OTA_PACKAGE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.OTA_PACKAGE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationRedisCache.java index b849d36f63..043f0f9437 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/relation/RelationRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/relation/RelationRedisCache.java @@ -20,15 +20,15 @@ import org.springframework.data.redis.connection.RedisConnectionFactory; import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.TBRedisCacheConfiguration; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("RelationCache") public class RelationRedisCache extends RedisTbTransactionalCache { public RelationRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.RELATIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.RELATIONS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java index 4f938bf690..46d5ef2964 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantExistsRedisCache.java @@ -21,15 +21,14 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; -import org.thingsboard.server.common.data.Tenant; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("TenantExistsCache") public class TenantExistsRedisCache extends RedisTbTransactionalCache { public TenantExistsRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileRedisCache.java index da83473f44..94855b2567 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantProfileRedisCache.java @@ -23,13 +23,13 @@ import org.thingsboard.server.cache.TBRedisCacheConfiguration; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.cache.RedisTbTransactionalCache; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; @ConditionalOnProperty(prefix = "cache", value = "type", havingValue = "redis") @Service("TenantProfileCache") public class TenantProfileRedisCache extends RedisTbTransactionalCache { public TenantProfileRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.TENANT_PROFILE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.TENANT_PROFILE_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java index 195e781755..1794da2674 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantRedisCache.java @@ -21,7 +21,7 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.cache.CacheSpecsMap; import org.thingsboard.server.cache.RedisTbTransactionalCache; import org.thingsboard.server.cache.TBRedisCacheConfiguration; -import org.thingsboard.server.cache.TbRedisSerializer; +import org.thingsboard.server.cache.TbFSTRedisSerializer; import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.Tenant; @@ -30,6 +30,6 @@ import org.thingsboard.server.common.data.Tenant; public class TenantRedisCache extends RedisTbTransactionalCache { public TenantRedisCache(TBRedisCacheConfiguration configuration, CacheSpecsMap cacheSpecsMap, RedisConnectionFactory connectionFactory) { - super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbRedisSerializer<>()); + super(CacheConstants.TENANTS_CACHE, cacheSpecsMap, connectionFactory, configuration, new TbFSTRedisSerializer<>()); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java index a6fefd3ead..45127d9517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/tenant/TenantServiceImpl.java @@ -175,12 +175,12 @@ public class TenantServiceImpl extends AbstractCachedEntityService&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_ARGS @@ -55,7 +55,7 @@ function additionalComposeQueueArgs() { ADDITIONAL_COMPOSE_QUEUE_ARGS="-f docker-compose.service-bus.yml" ;; *) - echo "Unknown Queue service value specified: '${TB_QUEUE_TYPE}'. Should be either kafka or confluent or aws-sqs or pubsub or rabbitmq or service-bus." >&2 + echo "Unknown Queue service TB_QUEUE_TYPE value specified in the .env file: '${TB_QUEUE_TYPE}'. Should be either 'kafka' or 'confluent' or 'aws-sqs' or 'pubsub' or 'rabbitmq' or 'service-bus'." >&2 exit 1 esac echo $ADDITIONAL_COMPOSE_QUEUE_ARGS @@ -85,7 +85,7 @@ function additionalComposeCacheArgs() { CACHE_COMPOSE_ARGS="-f docker-compose.redis-cluster.yml" ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac echo $CACHE_COMPOSE_ARGS @@ -102,7 +102,7 @@ function additionalStartupServices() { ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES postgres cassandra" ;; *) - echo "Unknown DATABASE value specified: '${DATABASE}'. Should be either postgres or hybrid." >&2 + echo "Unknown DATABASE value specified in the .env file: '${DATABASE}'. Should be either 'postgres' or 'hybrid'." >&2 exit 1 esac @@ -115,7 +115,7 @@ function additionalStartupServices() { ADDITIONAL_STARTUP_SERVICES="$ADDITIONAL_STARTUP_SERVICES redis-node-0 redis-node-1 redis-node-2 redis-node-3 redis-node-4 redis-node-5" ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac diff --git a/docker/docker-compose.hybrid.yml b/docker/docker-compose.hybrid.yml index 06f8b75b09..12ea4cd099 100644 --- a/docker/docker-compose.hybrid.yml +++ b/docker/docker-compose.hybrid.yml @@ -38,27 +38,23 @@ services: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-core2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-rule-engine1: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra tb-rule-engine2: env_file: - tb-node.hybrid.env depends_on: - - zookeeper - postgres - cassandra diff --git a/docker/docker-compose.kafka.yml b/docker/docker-compose.kafka.yml index 82fbb6ed62..e6fa0c489f 100644 --- a/docker/docker-compose.kafka.yml +++ b/docker/docker-compose.kafka.yml @@ -95,4 +95,4 @@ services: env_file: - queue-kafka.env depends_on: - - kafka \ No newline at end of file + - kafka diff --git a/docker/docker-compose.pubsub.yml b/docker/docker-compose.pubsub.yml index c03132d730..7c122d0835 100644 --- a/docker/docker-compose.pubsub.yml +++ b/docker/docker-compose.pubsub.yml @@ -59,4 +59,3 @@ services: tb-vc-executor2: env_file: - queue-pubsub.env - diff --git a/docker/docker-compose.rabbitmq.yml b/docker/docker-compose.rabbitmq.yml index d1acc32014..1f1cf1554c 100644 --- a/docker/docker-compose.rabbitmq.yml +++ b/docker/docker-compose.rabbitmq.yml @@ -58,4 +58,4 @@ services: - queue-rabbitmq.env tb-vc-executor2: env_file: - - queue-rabbitmq.env \ No newline at end of file + - queue-rabbitmq.env diff --git a/docker/docker-create-log-folders.sh b/docker/docker-create-log-folders.sh index 83257c0e66..6224b7040b 100755 --- a/docker/docker-create-log-folders.sh +++ b/docker/docker-create-log-folders.sh @@ -15,7 +15,7 @@ # limitations under the License. # -mkdir -p tb-node/log/ && sudo chown -R 799:799 tb-node/log/ +mkdir -p tb-node/log && sudo chown -R 799:799 tb-node/log mkdir -p tb-transports/coap/log && sudo chown -R 799:799 tb-transports/coap/log @@ -29,23 +29,25 @@ mkdir -p tb-transports/snmp/log && sudo chown -R 799:799 tb-transports/snmp/log mkdir -p tb-vc-executor/log && sudo chown -R 799:799 tb-vc-executor/log -mkdir -p tb-node/postgres/ && sudo chown -R 999:999 tb-node/postgres/ +mkdir -p tb-node/postgres && sudo chown -R 999:999 tb-node/postgres + +mkdir -p tb-node/cassandra && sudo chown -R 999:999 tb-node/cassandra source .env CACHE="${CACHE:-redis}" case $CACHE in redis) - mkdir -p tb-node/redis-data/ && sudo chown -R 1001:0 tb-node/redis-data/ + mkdir -p tb-node/redis-data && sudo chown -R 1001:1001 tb-node/redis-data ;; redis-cluster) - mkdir -p tb-node/redis-cluster-data-0/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-0/ - mkdir -p tb-node/redis-cluster-data-1/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-1/ - mkdir -p tb-node/redis-cluster-data-2/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-2/ - mkdir -p tb-node/redis-cluster-data-3/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-3/ - mkdir -p tb-node/redis-cluster-data-4/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-4/ - mkdir -p tb-node/redis-cluster-data-5/ && sudo chown -R 1001:0 tb-node/redis-cluster-data-5/ + mkdir -p tb-node/redis-cluster-data-0 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-0 + mkdir -p tb-node/redis-cluster-data-1 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-1 + mkdir -p tb-node/redis-cluster-data-2 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-2 + mkdir -p tb-node/redis-cluster-data-3 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-3 + mkdir -p tb-node/redis-cluster-data-4 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-4 + mkdir -p tb-node/redis-cluster-data-5 && sudo chown -R 1001:1001 tb-node/redis-cluster-data-5 ;; *) - echo "Unknown CACHE value specified: '${CACHE}'. Should be either redis or redis-cluster." >&2 + echo "Unknown CACHE value specified in the .env file: '${CACHE}'. Should be either 'redis' or 'redis-cluster'." >&2 exit 1 esac \ No newline at end of file diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 27e84c4d0b..9129c72796 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -50,9 +50,14 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then - docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES + docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES fi -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \ + tb-core1 diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 36e464f45d..769150c1f3 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS down -v +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + down -v diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 7993ab425b..9e5d08add9 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS up -d +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + up -d diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index ae8a3c0b81..5b09aea204 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -27,4 +27,6 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS stop +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS $ADDITIONAL_COMPOSE_MONITORING_ARGS \ + stop diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index a02f9e208a..027280635d 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -25,5 +25,9 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull $@ -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d --no-deps --build $@ +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull $@ +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d --no-deps --build $@ diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index c20543783b..260e14cf8b 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -48,8 +48,16 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS pull tb-core1 - -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS up -d $ADDITIONAL_STARTUP_SERVICES - -docker-compose -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} tb-core1 +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + pull \ + tb-core1 + +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + up -d $ADDITIONAL_STARTUP_SERVICES + +docker-compose \ + -f docker-compose.yml $ADDITIONAL_CACHE_ARGS $ADDITIONAL_COMPOSE_ARGS $ADDITIONAL_COMPOSE_QUEUE_ARGS \ + run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \ + tb-core1 diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 6802f19214..1fbf3755e6 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -47,6 +47,7 @@ public class ContainerTestSuite { private static final String SOURCE_DIR = "./../../docker/"; private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; + private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; private static DockerComposeContainer testContainer; @@ -77,7 +78,8 @@ public class ContainerTestSuite { } } - List composeFiles = new ArrayList<>(Arrays.asList(new File(targetDir + "docker-compose.yml"), + List composeFiles = new ArrayList<>(Arrays.asList( + new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), IS_HYBRID_MODE ? new File(targetDir + "docker-compose.hybrid.yml") @@ -85,11 +87,11 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose.kafka.yml"), IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.yml") - : new File("./../../docker/docker-compose.redis.yml"), + ? new File(targetDir + "docker-compose.redis-cluster.yml") + : new File(targetDir + "docker-compose.redis.yml"), IS_REDIS_CLUSTER - ? new File("./../../docker/docker-compose.redis-cluster.volumes.yml") - : new File("./../../docker/docker-compose.redis.volumes.yml"))); + ? new File(targetDir + "docker-compose.redis-cluster.volumes.yml") + : new File(targetDir + "docker-compose.redis.volumes.yml"))); if (IS_HYBRID_MODE) { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); @@ -107,7 +109,9 @@ public class ContainerTestSuite { .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) - .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); + .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) + .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))) + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(Duration.ofSeconds(400))); } catch (Exception e) { log.error("Failed to create test container", e); fail("Failed to create test container"); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 08cadbceb2..b627d2787a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -21,7 +21,11 @@ import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import java.io.File; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import java.util.stream.IntStream; diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index a08a5a28aa..32163ea01e 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -28,7 +28,7 @@ kafka: replication_factor: "1" acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge batch_size: "128" # for producer - linger_ms: "1" # for producer + linger_ms: "5" # for producer partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency requestTimeout: "30000" # The default value in kafkajs is: 30000 compression: "gzip" # gzip or uncompressed diff --git a/msa/vc-executor/src/main/conf/logback.xml b/msa/vc-executor/src/main/conf/logback.xml index d62cf2b3f5..c0f33852a6 100644 --- a/msa/vc-executor/src/main/conf/logback.xml +++ b/msa/vc-executor/src/main/conf/logback.xml @@ -35,6 +35,8 @@ + + diff --git a/msa/vc-executor/src/main/resources/logback.xml b/msa/vc-executor/src/main/resources/logback.xml index 572d093dde..e25a1995c1 100644 --- a/msa/vc-executor/src/main/resources/logback.xml +++ b/msa/vc-executor/src/main/resources/logback.xml @@ -28,6 +28,9 @@ + + + diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index a97c4cfd37..1344ea194a 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -168,6 +168,7 @@ queue: partitions: "${TB_QUEUE_VC_PARTITIONS:10}" poll-interval: "${TB_QUEUE_VC_INTERVAL_MS:25}" pack-processing-timeout: "${TB_QUEUE_VC_PACK_PROCESSING_TIMEOUT_MS:60000}" + msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:500000}" vc: # Pool size for handling export tasks diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java index f5752154c9..47d6af8e9c 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbAbstractGetAttributesNode.java @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonParseException; -import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.BooleanUtils; import org.thingsboard.rule.engine.api.TbContext; @@ -50,7 +49,6 @@ import static org.thingsboard.server.common.data.DataConstants.LATEST_TS; import static org.thingsboard.server.common.data.DataConstants.SERVER_SCOPE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; -@Slf4j public abstract class TbAbstractGetAttributesNode implements TbNode { private static ObjectMapper mapper = new ObjectMapper(); @@ -113,7 +111,6 @@ public abstract class TbAbstractGetAttributesNode> attributeKvEntryListFuture = ctx.getAttributesService().find(ctx.getTenantId(), entityId, scope, keys); return Futures.transform(attributeKvEntryListFuture, attributeKvEntryList -> { - log.warn("[{}][{}][{}][{}] Lookup attribute result: {}", ctx.getTenantId(), entityId, scope, keys, attributeKvEntryList); if (!CollectionUtils.isEmpty(attributeKvEntryList)) { List existingAttributesKvEntry = attributeKvEntryList.stream().filter(attributeKvEntry -> keys.contains(attributeKvEntry.getKey())).collect(Collectors.toList()); existingAttributesKvEntry.forEach(kvEntry -> msg.getMetaData().putValue(prefix + kvEntry.getKey(), kvEntry.getValueAsString())); diff --git a/transport/coap/src/main/conf/logback.xml b/transport/coap/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/coap/src/main/conf/logback.xml +++ b/transport/coap/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/coap/src/main/resources/logback.xml b/transport/coap/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/coap/src/main/resources/logback.xml +++ b/transport/coap/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/http/src/main/conf/logback.xml b/transport/http/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/http/src/main/conf/logback.xml +++ b/transport/http/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/http/src/main/resources/logback.xml b/transport/http/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/http/src/main/resources/logback.xml +++ b/transport/http/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/lwm2m/src/main/conf/logback.xml b/transport/lwm2m/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/lwm2m/src/main/conf/logback.xml +++ b/transport/lwm2m/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/lwm2m/src/main/resources/logback.xml b/transport/lwm2m/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/lwm2m/src/main/resources/logback.xml +++ b/transport/lwm2m/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/mqtt/src/main/conf/logback.xml b/transport/mqtt/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/mqtt/src/main/conf/logback.xml +++ b/transport/mqtt/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/mqtt/src/main/resources/logback.xml b/transport/mqtt/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/mqtt/src/main/resources/logback.xml +++ b/transport/mqtt/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/transport/snmp/src/main/conf/logback.xml b/transport/snmp/src/main/conf/logback.xml index df7fbf6e56..a769cbe364 100644 --- a/transport/snmp/src/main/conf/logback.xml +++ b/transport/snmp/src/main/conf/logback.xml @@ -37,6 +37,8 @@ + + diff --git a/transport/snmp/src/main/resources/logback.xml b/transport/snmp/src/main/resources/logback.xml index 240eccaa64..44ddb3d372 100644 --- a/transport/snmp/src/main/resources/logback.xml +++ b/transport/snmp/src/main/resources/logback.xml @@ -29,6 +29,9 @@ + + + diff --git a/ui-ngx/src/app/core/http/entities-version-control.service.ts b/ui-ngx/src/app/core/http/entities-version-control.service.ts index d7e892f1db..447f252fad 100644 --- a/ui-ngx/src/app/core/http/entities-version-control.service.ts +++ b/ui-ngx/src/app/core/http/entities-version-control.service.ts @@ -38,11 +38,10 @@ import { EntityType, entityTypeTranslations } from '@shared/models/entity-type.m import { select, Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { selectIsUserLoaded } from '@core/auth/auth.selectors'; -import { catchError, finalize, map, switchMap, takeWhile, tap } from 'rxjs/operators'; +import { catchError, finalize, switchMap, takeWhile, tap } from 'rxjs/operators'; import { TranslateService } from '@ngx-translate/core'; import { DomSanitizer, SafeHtml } from '@angular/platform-browser'; import { ActionLoadFinish, ActionLoadStart } from '@core/interceptors/load.actions'; -import { NULL_UUID } from '@shared/models/id/has-uuid'; @Injectable({ providedIn: 'root' @@ -184,6 +183,9 @@ export class EntitiesVersionControlService { (this.translate.instant(entityTypeTranslations.get(entityLoadError.target.entityType).type) as string).toLowerCase(); messageArgs.targetEntityId = entityLoadError.target.id; break; + case EntityLoadErrorType.RUNTIME: + messageArgs.message = entityLoadError.message; + break; } return this.sanitizer.bypassSecurityTrustHtml(this.translate.instant(messageId, messageArgs)); } diff --git a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html index b04ab515ae..6529b4b97b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/auto-commit-settings.component.html @@ -32,64 +32,63 @@
admin.auto-commit-entities
-
-
- - -
- -
-
-
- - - -
- - -
- -
-
- - -
- - -
-
-
- - {{ 'version-control.export-credentials' | translate }} - - - {{ 'version-control.export-attributes' | translate }} - - - {{ 'version-control.export-relations' | translate }} - +
+ + +
+ +
+
+
+ + + +
+ + +
+ +
+
+ + +
+ +
+
+ + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
- - -
+
+ +
-
+

{{ 'version-control.create-entities-version' | translate }}

@@ -35,7 +35,8 @@ version-control.version-name - + {{ 'version-control.version-name-required' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts index 5348b4b340..8b53bdaeac 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-create.component.ts @@ -20,7 +20,9 @@ import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { ComplexVersionCreateRequest, createDefaultEntityTypesVersionCreate, - SyncStrategy, syncStrategyHintMap, syncStrategyTranslationMap, + SyncStrategy, + syncStrategyHintMap, + syncStrategyTranslationMap, VersionCreateRequestType, VersionCreationResult } from '@shared/models/vc.models'; @@ -82,7 +84,7 @@ export class ComplexVersionCreateComponent extends PageComponent implements OnIn ngOnInit(): void { this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], - versionName: [null, [Validators.required]], + versionName: [null, [Validators.required, Validators.pattern(/(?:.|\s)*\S(&:.|\s)*/)]], syncStrategy: [SyncStrategy.MERGE, Validators.required], entityTypes: [createDefaultEntityTypesVersionCreate(), []], }); diff --git a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html index 05f16b3e2a..2df32ec5e2 100644 --- a/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/complex-version-load.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'version-control.restore-entities-from-version' | translate: {versionName} }}

diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html index 928b7a7668..89b2a67c4d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-types-version-create.component.html @@ -19,81 +19,79 @@
version-control.entities-to-export
-
-
- - -
- -
-
{{ entityTypeText(entityTypeFormGroup) }}
-
- - - -
- - -
- -
- - -
- - version-control.sync-strategy - - - {{ 'version-control.default' | translate }} - - - {{syncStrategyTranslations.get(strategy) | translate}} - - - -
- - {{ 'version-control.export-credentials' | translate }} - - - {{ 'version-control.export-attributes' | translate }} - - - {{ 'version-control.export-relations' | translate }} - -
-
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
-
- - {{ 'version-control.all-entities' | translate }} - - - + + + +
+ + +
+ +
+ + +
+ + version-control.sync-strategy + + + {{ 'version-control.default' | translate }} + + + {{syncStrategyTranslations.get(strategy) | translate}} + + + +
+ + {{ 'version-control.export-credentials' | translate }} + + + {{ 'version-control.export-attributes' | translate }} + + + {{ 'version-control.export-relations' | translate }} + +
- - -
+
+ + {{ 'version-control.all-entities' | translate }} + + + +
+
+ +
version-control.entities-to-restore
-
-
- - -
- -
-
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + +
+ +
+
{{ entityTypeText(entityTypeFormGroup) }}
+
+ + + +
+ + +
+ +
+ + +
+
+ + {{ 'version-control.remove-other-entities' | translate }} + + + {{ 'version-control.find-existing-entity-by-name' | translate }} +
- - - -
- - -
- -
- - -
-
- - {{ 'version-control.remove-other-entities' | translate }} - - - {{ 'version-control.find-existing-entity-by-name' | translate }} - -
-
- - {{ 'version-control.load-credentials' | translate }} - - - {{ 'version-control.load-attributes' | translate }} - - - {{ 'version-control.load-relations' | translate }} - -
+
+ + {{ 'version-control.load-credentials' | translate }} + + + {{ 'version-control.load-attributes' | translate }} + + + {{ 'version-control.load-relations' | translate }} +
- - -
+
+ +
-
+

{{ 'version-control.create-entity-version' | translate }}

@@ -34,7 +34,8 @@ version-control.version-name - + {{ 'version-control.version-name-required' | translate }} diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts index 6706d80c71..0b8812f94d 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-create.component.ts @@ -80,7 +80,7 @@ export class EntityVersionCreateComponent extends PageComponent implements OnIni this.createVersionFormGroup = this.fb.group({ branch: [this.branch, [Validators.required]], versionName: [this.translate.instant('version-control.default-create-entity-version-name', - {entityName: this.entityName}), [Validators.required]], + {entityName: this.entityName}), [Validators.required, Validators.pattern(/(?:.|\s)*\S(&:.|\s)*/)]], saveRelations: [false, []], saveAttributes: [true, []], saveCredentials: [true, []] diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss index cf0bf939b7..e3efdc871b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.scss @@ -19,6 +19,7 @@ .entity-version-diff-view { position: relative; &:not(.tb-fullscreen) { + max-height: 95vh; &.content-ready { width: 600px; @media #{$mat-gt-sm} { diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts index 29955af18e..eb9fe8118b 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-diff.component.ts @@ -161,7 +161,7 @@ export class EntityVersionDiffComponent extends PageComponent implements OnInit, setTimeout(() => { this.diffCount = this.differ.getNumDiffs(); this.updateHasNextAndPrevious(); - }); + }, 2); }); }); }); diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html index b096c1907f..cbd2c72c31 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-version-restore.component.html @@ -15,7 +15,7 @@ limitations under the License. --> -
+

{{ 'version-control.restore-entity-from-version' | translate: {versionName} }}

diff --git a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html index b6accc40dc..2f3abbb12c 100644 --- a/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html +++ b/ui-ngx/src/app/modules/home/components/vc/entity-versions-table.component.html @@ -31,32 +31,36 @@
- - - - +
+ + +
+ + +
+
@@ -70,7 +74,7 @@   + placeholder="{{ 'version-control.search' | translate }}"/>