From fb161730fb28fd4d53907426dd2038e5401f4b48 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 21 Feb 2025 18:38:26 +0200 Subject: [PATCH] added msa tests for edqs --- .../controller/EntityQueryController.java | 7 + .../entitiy/EdqsEntityServiceTest.java | 49 +++++ .../service/entitiy/EntityServiceTest.java | 3 +- .../common/data/edqs/fields/FieldsUtil.java | 4 +- .../edqs/repo/InMemoryEdqRepository.java | 9 +- .../server/dao/entity/BaseEntityService.java | 4 +- .../dao/sql/query/DummyEdqsService.java | 1 + docker/.env | 1 + docker/compose-utils.sh | 12 ++ docker/docker-compose.edqs.yml | 33 +++ docker/docker-remove-services.sh | 4 +- docker/docker-start-services.sh | 4 +- docker/docker-stop-services.sh | 4 +- docker/edqs.env | 7 + docker/tb-node.env | 2 + edqs/src/main/resources/edqs.yml | 5 + .../server/msa/ContainerTestSuite.java | 11 +- .../server/msa/TestRestClient.java | 100 ++++++++- .../server/msa/ThingsBoardDbInstaller.java | 3 +- .../msa/edqs/EdqsEntityDataQueryTest.java | 202 ++++++++++++++++++ .../server/msa/ui/utils/EntityPrototypes.java | 32 +++ 21 files changed, 477 insertions(+), 20 deletions(-) create mode 100644 docker/docker-compose.edqs.yml create mode 100644 docker/edqs.env create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index f28d485b30..9c2c70d019 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -20,6 +20,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -144,4 +145,10 @@ public class EntityQueryController extends BaseController { edqsService.processSystemRequest(request); } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @GetMapping("/edqs/enabled") + public boolean isEdqsApiEnabled() { + return edqsService.isApiEnabled(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index cb377e0431..61f246ca3e 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -15,20 +15,35 @@ */ package org.thingsboard.server.service.entitiy; +import com.google.common.collect.Lists; import org.junit.Before; +import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.awaitility.Awaitility.await; @@ -51,6 +66,40 @@ public class EdqsEntityServiceTest extends EntityServiceTest { await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsService.isApiEnabled()); } + // sql implementation has a bug with data duplication, edqs implementation returns correct value + @Override + @Test + public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { + List buildings = new ArrayList<>(); + List apartments = new ArrayList<>(); + Map> entityNameByTypeMap = new HashMap<>(); + Map childParentRelationMap = new HashMap<>(); + createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap); + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setMultiRoot(true); + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(buildings.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.FROM); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + countByQueryAndCheck(countQuery, 63); + + filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 27); + + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.TO); + filter.setFilters(Lists.newArrayList( + new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), + new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 3); + + deviceService.deleteDevicesByTenantId(tenantId); + assetService.deleteAssetsByTenantId(tenantId); + } + @Override protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index a559eca6ce..44e636b04f 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -462,7 +462,6 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } - // fails for sql implementation until we fix the issue with the relation query @Test public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { List buildings = new ArrayList<>(); @@ -489,7 +488,7 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - countByQueryAndCheck(countQuery, 3); + countByQueryAndCheck(countQuery, 9); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index 6543a6d779..8d939e07e5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -240,7 +240,7 @@ public class FieldsUtil { .build(); } - private static AssetProfileFields toFields(DeviceProfile entity) { + private static AssetProfileFields toFields(AssetProfile entity) { return AssetProfileFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) @@ -250,7 +250,7 @@ public class FieldsUtil { .build(); } - private static DeviceProfileFields toFields(AssetProfile entity) { + private static DeviceProfileFields toFields(DeviceProfile entity) { return DeviceProfileFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java index 7736f303ac..1b6903fc3c 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/InMemoryEdqRepository.java @@ -61,14 +61,7 @@ public class InMemoryEdqRepository implements EdqRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); - long result = 0; - if (TenantId.SYS_TENANT_ID.equals(tenantId)) { - for (TenantRepo repo : repos.values()) { - result += repo.countEntitiesByQuery(customerId, query, ignorePermissionCheck); - } - } else { - result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); - } + long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; log.info("countEntitiesByQuery: {} ms", timingMs); return result; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 3f941b776c..7705614a05 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -95,7 +95,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); - if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always EdqsRequest request = EdqsRequest.builder() .entityCountQuery(query) .build(); @@ -112,7 +112,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); - if (edqsService.isApiEnabled() && validForEdqs(query)) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always + if (edqsService.isApiEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { // TODO: separate boolean param whether to use in dashboards; but sync to edqs - always EdqsRequest request = EdqsRequest.builder() .entityDataQuery(query) .build(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java index b69aa52ec5..14ab995a45 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DummyEdqsService.java @@ -16,6 +16,7 @@ package org.thingsboard.server.dao.sql.query; import com.google.common.util.concurrent.ListenableFuture; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.ObjectType; diff --git a/docker/.env b/docker/.env index 37c9768296..38724263fc 100644 --- a/docker/.env +++ b/docker/.env @@ -14,6 +14,7 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor +EDQS_DOCKER_NAME=edqs TB_VERSION=latest diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index aa5a9db08d..9411c22b7e 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -128,6 +128,18 @@ function additionalStartupServices() { echo $ADDITIONAL_STARTUP_SERVICES } +function additionalComposeEdqsArgs() { + source .env + + if [ "$EDQS_ENABLED" = true ] + then + ADDITIONAL_COMPOSE_EDQS_ARGS="-f docker-compose.edqs.yml" + echo ADDITIONAL_COMPOSE_EDQS_ARGS + else + echo "" + fi +} + function permissionList() { PERMISSION_LIST=" 799 799 tb-node/log diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml new file mode 100644 index 0000000000..f989c12a2e --- /dev/null +++ b/docker/docker-compose.edqs.yml @@ -0,0 +1,33 @@ +# +# Copyright © 2016-2024 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + edqs-1: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + depends_on: + - zookeeper + edqs-2: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + depends_on: + - zookeeper diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index c77499f833..d7fd891925 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ down -v" case $COMPOSE_VERSION in diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 42af042108..3d4479c001 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -29,10 +29,12 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + checkFolders --create || exit $? COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d" case $COMPOSE_VERSION in diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 2a753cc348..b7037fce69 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS}\ stop" case $COMPOSE_VERSION in diff --git a/docker/edqs.env b/docker/edqs.env new file mode 100644 index 0000000000..3c91c92e54 --- /dev/null +++ b/docker/edqs.env @@ -0,0 +1,7 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 +TB_KAFKA_SERVERS=kafka:9092 + +TB_EDQS_STATS_ENABLED=false +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus diff --git a/docker/tb-node.env b/docker/tb-node.env index 85a60eb51a..d40ca66621 100644 --- a/docker/tb-node.env +++ b/docker/tb-node.env @@ -9,3 +9,5 @@ HTTP_LOG_CONTROLLER_ERROR_STACK_TRACE=false METRICS_ENABLED=true METRICS_ENDPOINTS_EXPOSE=prometheus + +TB_EDQS_MODE=remote \ No newline at end of file diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index b67f2f5f9e..a55391451e 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -60,6 +60,11 @@ queue: poll_interval: "${TB_EDQS_POLL_INTERVAL_MS:125}" max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:10000}" + stats: + # Enable/disable statistics for EDQS service + enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Statistics printing interval for EDQS + print-interval-ms: "${TB_EDQS_STATS_PRINT_INTERVAL_MS:60000}" kafka: # Kafka Bootstrap nodes in "host:port" format bootstrap.servers: "${TB_KAFKA_SERVERS:localhost:9092}" diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index bbaeabe907..d22861e286 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -114,6 +114,7 @@ public class ContainerTestSuite { List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.edqs.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.mosquitto.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), @@ -162,6 +163,12 @@ public class ContainerTestSuite { composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); } + // to trigger edqs synchronization + if (true) { + addToFile(targetDir, "tb-node.env", + Map.of("TB_EDQS_SYNC_ENABLED", "true")); + } + testContainer = new DockerComposeContainerImpl<>(composeFiles) .withPull(false) .withLocalCompose(true) @@ -180,7 +187,9 @@ public class ContainerTestSuite { .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("edqs-1", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("edqs-2", Wait.forHttp("/api/edqs/ready").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index d137f8ed59..3c61c65ef1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -28,6 +28,7 @@ import io.restassured.internal.ValidatableResponseImpl; import io.restassured.path.json.JsonPath; import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; import org.thingsboard.server.common.data.Device; @@ -35,6 +36,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; @@ -56,6 +58,9 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rpc.Rpc; @@ -66,7 +71,6 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import static io.restassured.RestAssured.given; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; @@ -110,6 +114,37 @@ public class TestRestClient { requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); } + public JsonNode getActivateRequest(String password) { + ObjectNode response = given().spec(requestSpec).get("/api/noauth/activate?activateToken={activateToken}", token) + .then() + .statusCode(HTTP_OK) + .extract() + .as(ObjectNode.class); + return JacksonUtil.newObjectNode() + .put("activateToken", this.token) + .put("password", password); + } + + public void activateAndLoginAsUser(JsonNode activateRequest) { + ObjectNode tokenInfo = given().spec(requestSpec).body(activateRequest) + .post("/api/noauth/activate") + .then() + .extract() + .as(ObjectNode.class); + token = tokenInfo.get("token").asText(); + refreshToken = tokenInfo.get("refreshToken").asText(); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + public Tenant postTenant(Tenant tenant) { + return given().spec(requestSpec).body(tenant) + .post("/api/tenant") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Tenant.class); + } + public Device postDevice(String accessToken, Device device) { return given().spec(requestSpec).body(device) .pathParams("accessToken", accessToken) @@ -479,6 +514,28 @@ public class TestRestClient { .as(User.class); } + public UserId createUserAndLogin(User user, String password) { + UserId userId = postUser(user).getId(); + getUserToken(userId.getId().toString()); + return userId; + } + + public void getUserToken(String id) { + ObjectNode tokenInfo = given().spec(requestSpec) + .get("/api/user/" + id + "/token") + .then() + .extract() + .as(ObjectNode.class); + token = tokenInfo.get("token").asText(); + refreshToken = tokenInfo.get("refreshToken").asText(); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + protected void resetTokens() { + this.token = null; + this.refreshToken = null; + } + public void deleteUser(UserId userId) { given().spec(requestSpec) .delete("/api/user/{userId}", userId.getId()) @@ -643,4 +700,45 @@ public class TestRestClient { } return urlParams; } + + public PageData postEntityDataQuery(EntityDataQuery entityDataQuery) { + return given().spec(requestSpec).body(entityDataQuery) + .post("/api/entitiesQuery/find") + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef<>() {}); + } + + public Long postCountDataQuery(EntityCountQuery entityCountQuery) { + return given().spec(requestSpec).body(entityCountQuery) + .post("/api/entitiesQuery/count") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Long.class); + } + + public Boolean isEdqsApiEnabled() { + return given().spec(requestSpec) + .get("/api/edqs/enabled") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Boolean.class); + } + + public void assignDeviceToCustomer(CustomerId customerId, DeviceId id) { + given().spec(requestSpec) + .post("/api/customer/" + customerId.getId().toString() + "/device/" + id.getId().toString()) + .then() + .statusCode(HTTP_OK); + } + + public void deleteTenant(TenantId tenantId) { + given().spec(requestSpec) + .delete("/api/tenant/" + tenantId.getId().toString()) + .then() + .statusCode(HTTP_OK); + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index aaa4f001c6..75f9779916 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -220,7 +220,8 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true -e TB_EDQS_SYNC_ENABLED=false " + + "tb-core1"); dockerCompose.invokeCompose(); } finally { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java new file mode 100644 index 0000000000..1317011cf0 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -0,0 +1,202 @@ +/** + * Copyright © 2016-2024 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.edqs; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.ui.utils.EntityPrototypes; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomer; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomerAdmin; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultDeviceProfile; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultTenantAdmin; + +@DisableUIListeners +public class EdqsEntityDataQueryTest extends AbstractContainerTest { + + private TenantId tenantId; + private CustomerId customerId; + private TenantId tenantId2; + private CustomerId customerId2; + private UserId tenantAdminId; + private UserId customerUserId; + private UserId tenant2AdminId; + private UserId customer2UserId; + private final List tenantDevices = new ArrayList<>(); + private final List tenant2Devices = new ArrayList<>(); + private final String deviceProfile = "LoRa-" + RandomStringUtils.randomAlphabetic(10); + + @BeforeClass + public void beforeClass() throws Exception { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> testRestClient.isEdqsApiEnabled()); + + tenantId = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenantAdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId, "tenantAdmin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenantDevices, 97); + customerId = testRestClient.postCustomer(defaultCustomer(tenantId, "Customer")).getId(); + customerUserId = testRestClient.postUser(defaultCustomerAdmin(tenantId, customerId, "customerUser@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId, tenantDevices, 12); + + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + tenantId2 = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenant2AdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId2, "tenant2Admin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenant2Devices, 97); + customerId2 = testRestClient.postCustomer(defaultCustomer(tenantId2, "Customer")).getId(); + customer2UserId = testRestClient.postUser(defaultCustomerAdmin(tenantId2, customerId2, "customer2User@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId2, tenant2Devices, 12); + } + + @BeforeMethod + public void beforeMethod() { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + } + + @AfterClass + public void afterClass() { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.deleteTenant(tenantId); + testRestClient.deleteTenant(tenantId2); + } + + @Test + public void testSysAdminCountEntitiesByQuery() { + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery query = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).compareTo(97L * 2) >= 0); + + testRestClient.getUserToken(tenantAdminId.getId().toString()); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getUserToken(tenant2AdminId.getId().toString()); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + } + + @Test + public void testRetrieveTenantDevicesByDeviceTypeFilter() { + // login tenant admin + testRestClient.getUserToken(tenantAdminId.getId().toString()); + checkUserDevices(tenantDevices); + + // login customer user + testRestClient.getUserToken(customerUserId.getId().toString()); + checkUserDevices(tenantDevices.subList(0, 12)); + + // login other tenant admin + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getUserToken(tenant2AdminId.getId().toString()); + checkUserDevices(tenant2Devices); + } + + private void checkUserDevices(List devices) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of(deviceProfile)); + filter.setDeviceNameFilter(""); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestFields = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestFields, null); + + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery countQuery = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(countQuery).intValue() == devices.size()); + + PageData result = testRestClient.postEntityDataQuery(query); + assertThat(result.getTotalElements()).isEqualTo(devices.size()); + List retrievedDevices = result.getData(); + + assertThat(retrievedDevices).hasSize(10); + List retrievedDeviceNames = retrievedDevices.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).toList(); + assertThat(retrievedDeviceNames).containsExactlyInAnyOrderElementsOf(devices.stream().map(Device::getName).toList().subList(0, 10)); + } + + private String createDevices(String deviceType, List tenantDevices, int deviceCount) throws InterruptedException { + String prefix = StringUtils.randomAlphabetic(5); + for (int i = 0; i < deviceCount; i++) { + Device device = new Device(); + device.setName(prefix + "Device" + i); + device.setType(deviceType); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + //TO make sure devices have different created time + Thread.sleep(1); + String token = RandomStringUtils.randomAlphabetic(10); + Device saved = testRestClient.postDevice(token, device); + tenantDevices.add(saved); + + // save timeseries data + testRestClient.postTelemetry(token, createDeviceTelemetry(i)); + } + return deviceType; + } + + private void assignDevicesToCustomer(CustomerId customerId, List devices, int deviceCount) { + for (int i = 0; i < deviceCount; i++) { + Device device = devices.get(i); + testRestClient.assignDeviceToCustomer(customerId, device.getId()); + } + } + + protected ObjectNode createDeviceTelemetry(int temperature) { + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put("temperature", temperature); + return objectNode; + } + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java index d16cbe35e7..e1c684ddf9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -37,12 +38,26 @@ import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfilePr import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.security.Authority; public class EntityPrototypes { + public static Tenant defaultTenantPrototype(String tenantName) { + Tenant tenant = new Tenant(); + tenant.setTitle(tenantName); + return tenant; + } + + public static Customer defaultCustomer(TenantId tenantId, String title) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(title); + return customer; + } + public static Customer defaultCustomerPrototype(String entityName) { Customer customer = new Customer(); customer.setTitle(entityName); @@ -169,6 +184,23 @@ public class EntityPrototypes { return user; } + public static User defaultTenantAdmin(TenantId tenantId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setEmail(email); + user.setAuthority(Authority.TENANT_ADMIN); + return user; + } + + public static User defaultCustomerAdmin(TenantId tenantId, CustomerId customerId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setCustomerId(customerId); + user.setEmail(email); + user.setAuthority(Authority.CUSTOMER_USER); + return user; + } + public static User defaultUser(String email, CustomerId customerId, String name) { User user = new User(); user.setEmail(email);