diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index db2f15a4a1..7fd0b12077 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -20,6 +20,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; @@ -40,6 +41,7 @@ import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.msg.edqs.EdqsApiService; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -60,6 +62,8 @@ public class EntityQueryController extends BaseController { private EntityQueryService entityQueryService; @Autowired private EdqsService edqsService; + @Autowired + private EdqsApiService edqsApiService; private static final int MAX_PAGE_SIZE = 100; @@ -144,4 +148,10 @@ public class EntityQueryController extends BaseController { edqsService.processSystemRequest(request); } + @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") + @GetMapping("/edqs/enabled") + public boolean isEdqsApiEnabled() { + return edqsApiService.isEnabled(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java index 0be4403aad..79e0e60983 100644 --- a/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java +++ b/application/src/main/java/org/thingsboard/server/service/edqs/EdqsSyncService.java @@ -166,7 +166,7 @@ public abstract class EdqsSyncService { if (entityIdInfo != null) { process(entityIdInfo.tenantId(), RELATION, relation.toData()); } else { - log.info("Relation from entity not found: " + relation.getFromType() + " " + relation.getFromId()); + log.info("Relation from id not found: {} ", relation); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java index f6ce2a6a6b..f6b4067543 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/TbAlarmDataSubCtx.java @@ -39,6 +39,7 @@ import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; import org.thingsboard.server.dao.model.ModelConstants; +import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmDataUpdate; @@ -359,7 +360,7 @@ public class TbAlarmDataSubCtx extends TbAbstractDataSubCtx { EntityDataSortOrder sortOrder = query.getPageLink().getSortOrder(); EntityDataSortOrder entitiesSortOrder; if (sortOrder == null || sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { - entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, ModelConstants.CREATED_TIME_PROPERTY)); + entitiesSortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, EntityKeyMapping.CREATED_TIME)); } else { entitiesSortOrder = sortOrder; } diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java index f046aedab0..011399e883 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityQueryControllerTest.java @@ -713,7 +713,8 @@ public class EntityQueryControllerTest extends AbstractControllerTest { // all devices with ownerName = TEST TENANT EntityCountQuery query = new EntityCountQuery(filter, List.of(activeAlarmTimeFilter, tenantOwnerNameFilter)); - countByQueryAndCheck(query, numOfDevices); + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> countByQuery(query), + result -> result == numOfDevices); // all devices with ownerName = TEST TENANT EntityCountQuery activeAlarmTimeToLongQuery = new EntityCountQuery(filter, List.of(activeAlarmTimeToLongFilter, tenantOwnerNameFilter)); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java index 25d7407bb6..f4cdf3038b 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EdqsEntityServiceTest.java @@ -15,20 +15,36 @@ */ package org.thingsboard.server.service.entitiy; +import com.google.common.collect.Lists; import org.junit.Before; +import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.test.context.TestPropertySource; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.asset.Asset; import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.IdBased; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.RelationsQueryFilter; +import org.thingsboard.server.common.data.relation.EntitySearchDirection; +import org.thingsboard.server.common.data.relation.RelationEntityTypeFilter; import org.thingsboard.server.common.msg.edqs.EdqsApiService; import org.thingsboard.server.dao.service.DaoSqlTest; import org.thingsboard.server.edqs.util.EdqsRocksDb; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import static org.awaitility.Awaitility.await; @@ -51,12 +67,52 @@ public class EdqsEntityServiceTest extends EntityServiceTest { await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> edqsApiService.isEnabled()); } + // sql implementation has a bug with data duplication, edqs implementation returns correct value + @Override + @Test + public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { + List buildings = new ArrayList<>(); + List apartments = new ArrayList<>(); + Map> entityNameByTypeMap = new HashMap<>(); + Map childParentRelationMap = new HashMap<>(); + createMultiRootHierarchy(buildings, apartments, entityNameByTypeMap, childParentRelationMap); + + RelationsQueryFilter filter = new RelationsQueryFilter(); + filter.setMultiRoot(true); + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(buildings.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.FROM); + + EntityCountQuery countQuery = new EntityCountQuery(filter); + countByQueryAndCheck(countQuery, 63); + + filter.setFilters(Collections.singletonList(new RelationEntityTypeFilter("AptToHeat", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 27); + + filter.setMultiRootEntitiesType(EntityType.ASSET); + filter.setMultiRootEntityIds(apartments.stream().map(IdBased::getId).map(d -> d.getId().toString()).collect(Collectors.toSet())); + filter.setDirection(EntitySearchDirection.TO); + filter.setFilters(Lists.newArrayList( + new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), + new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); + countByQueryAndCheck(countQuery, 3); + + deviceService.deleteDevicesByTenantId(tenantId); + assetService.deleteAssetsByTenantId(tenantId); + } + @Override protected PageData findByQueryAndCheck(CustomerId customerId, EntityDataQuery query, long expectedResultSize) { return await().atMost(15, TimeUnit.SECONDS).until(() -> findByQuery(customerId, query), result -> result.getTotalElements() == expectedResultSize); } + @Override + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + return await().atMost(15, TimeUnit.SECONDS).until(() -> findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetries), + loadedTelemetry -> loadedTelemetry.containsAll(expectedTelemetries)); + } + @Override protected long countByQueryAndCheck(EntityCountQuery countQuery, int expectedResult) { return countByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), countQuery, expectedResult); diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index d208867457..dc1f46eb3c 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -462,7 +462,6 @@ public class EntityServiceTest extends AbstractControllerTest { deviceService.deleteDevicesByTenantId(tenantId); } - // fails for sql implementation until we fix the issue with the relation query @Test public void testCountHierarchicalEntitiesByMultiRootQuery() throws InterruptedException { List buildings = new ArrayList<>(); @@ -489,7 +488,7 @@ public class EntityServiceTest extends AbstractControllerTest { filter.setFilters(Lists.newArrayList( new RelationEntityTypeFilter("buildingToApt", Collections.singletonList(EntityType.ASSET)), new RelationEntityTypeFilter("AptToEnergy", Collections.singletonList(EntityType.DEVICE)))); - countByQueryAndCheck(countQuery, 3); + countByQueryAndCheck(countQuery, 9); deviceService.deleteDevicesByTenantId(tenantId); assetService.deleteAssetsByTenantId(tenantId); @@ -1572,47 +1571,19 @@ public class EntityServiceTest extends AbstractControllerTest { for (EntityKeyType currentAttributeKeyType : attributesEntityTypes) { List latestValues = Collections.singletonList(new EntityKey(currentAttributeKeyType, "temperature")); EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, null); - PageData data = findByQueryAndCheck(query, 67); - List loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(67, loadedEntities.size()); - List loadedTemperatures = new ArrayList<>(); - for (Device device : devices) { - loadedTemperatures.add(loadedEntities.stream().filter(entityData -> entityData.getEntityId().equals(device.getId())).findFirst().orElse(null) - .getLatest().get(currentAttributeKeyType).get("temperature").getValue()); - } - List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - assertThat(loadedTemperatures).containsExactlyInAnyOrderElementsOf(deviceTemperatures); + List deviceTemperatures = temperatures.stream().map(aLong -> Long.toString(aLong)).toList(); + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", deviceTemperatures); pageLink = new EntityDataPageLink(10, 0, null, sortOrder); KeyFilter highTemperatureFilter = createNumericKeyFilter("temperature", currentAttributeKeyType, NumericFilterPredicate.NumericOperation.GREATER, 45); List keyFiltersHighTemperature = Collections.singletonList(highTemperatureFilter); query = new EntityDataQuery(filter, pageLink, entityFields, latestValues, keyFiltersHighTemperature); - data = findByQueryAndCheck(query, highTemperatures.size()); - - loadedEntities = new ArrayList<>(data.getData()); - while (data.hasNext()) { - query = query.next(); - data = findByQuery(query); - loadedEntities.addAll(data.getData()); - } - Assert.assertEquals(highTemperatures.size(), loadedEntities.size()); - - List loadedHighTemperatures = loadedEntities.stream().map(entityData -> - entityData.getLatest().get(currentAttributeKeyType).get("temperature").getValue()).collect(Collectors.toList()); - List deviceHighTemperatures = highTemperatures.stream().map(aLong -> Long.toString(aLong)).collect(Collectors.toList()); - - assertThat(loadedHighTemperatures).containsExactlyInAnyOrderElementsOf(deviceHighTemperatures); + findByQueryAndCheckTelemetry(query, currentAttributeKeyType, "temperature", highTemperatures.stream().map(Object::toString).toList()); } deviceService.deleteDevicesByTenantId(tenantId); } - @Test public void testBuildNumericPredicateQueryOperations() throws ExecutionException, InterruptedException { @@ -2520,7 +2491,7 @@ public class EntityServiceTest extends AbstractControllerTest { findByQueryAndCheck(new CustomerId(EntityId.NULL_UUID), query, 0); } - private PageData findByQuery(EntityDataQuery query) { + protected PageData findByQuery(EntityDataQuery query) { return findByQuery(new CustomerId(CustomerId.NULL_UUID), query); } @@ -2528,7 +2499,7 @@ public class EntityServiceTest extends AbstractControllerTest { return entityService.findEntityDataByQuery(tenantId, customerId, query); } - private PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { + protected PageData findByQueryAndCheck(EntityDataQuery query, long expectedResultSize) { return findByQueryAndCheck(new CustomerId(CustomerId.NULL_UUID), query, expectedResultSize); } @@ -2538,6 +2509,23 @@ public class EntityServiceTest extends AbstractControllerTest { return result; } + protected List findByQueryAndCheckTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetry) { + List entitiesTelemetry = findEntitiesTelemetry(query, entityKeyType, key, expectedTelemetry); + assertThat(entitiesTelemetry).containsExactlyInAnyOrderElementsOf(expectedTelemetry); + return entitiesTelemetry; + } + + protected List findEntitiesTelemetry(EntityDataQuery query, EntityKeyType entityKeyType, String key, List expectedTelemetries) { + PageData data = findByQueryAndCheck(query, expectedTelemetries.size()); + List loadedEntities = new ArrayList<>(data.getData()); + while (data.hasNext()) { + query = query.next(); + data = findByQuery(query); + loadedEntities.addAll(data.getData()); + } + return loadedEntities.stream().map(entityData -> entityData.getLatest().get(entityKeyType).get(key).getValue()).toList(); + } + protected long countByQuery(CustomerId customerId, EntityCountQuery query) { return entityService.countEntitiesByQuery(tenantId, customerId, query); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java index e25ef57a44..51c631fe4f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/ObjectType.java @@ -60,15 +60,14 @@ public enum ObjectType { LATEST_TS_KV; public static final Set edqsTenantTypes = EnumSet.of( - TENANT, TENANT_PROFILE, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, + TENANT, CUSTOMER, DEVICE_PROFILE, DEVICE, ASSET_PROFILE, ASSET, EDGE, ENTITY_VIEW, USER, DASHBOARD, RULE_CHAIN, WIDGET_TYPE, WIDGETS_BUNDLE, API_USAGE_STATE, QUEUE_STATS ); public static final Set edqsTypes = EnumSet.copyOf(edqsTenantTypes); - public static final Set edqsSystemTypes = EnumSet.of(TENANT, TENANT_PROFILE, USER, DASHBOARD, + public static final Set edqsSystemTypes = EnumSet.of(TENANT, USER, DASHBOARD, API_USAGE_STATE, ATTRIBUTE_KV, LATEST_TS_KV); public static final Set unversionedTypes = EnumSet.of( - QUEUE_STATS, // created once, never updated - TENANT_PROFILE // only for total count calculation + QUEUE_STATS // created once, never updated ); static { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index ce9873b5d6..a36514248c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -240,7 +240,7 @@ public class FieldsUtil { .build(); } - private static AssetProfileFields toFields(DeviceProfile entity) { + private static AssetProfileFields toFields(AssetProfile entity) { return AssetProfileFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) @@ -250,7 +250,7 @@ public class FieldsUtil { .build(); } - private static DeviceProfileFields toFields(AssetProfile entity) { + private static DeviceProfileFields toFields(DeviceProfile entity) { return DeviceProfileFields.builder() .id(entity.getUuidId()) .createdTime(entity.getCreatedTime()) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java index f1195af284..342e2974a4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/TenantFields.java @@ -56,4 +56,8 @@ public class TenantFields extends AbstractEntityFields { this.region = region; } + @Override + public UUID getTenantId() { + return getId(); + } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java index 5e4818b508..1deaca83a7 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java @@ -61,14 +61,7 @@ public class DefaultEdqsRepository implements EdqsRepository { @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query, boolean ignorePermissionCheck) { long startNs = System.nanoTime(); - long result = 0; - if (!tenantId.isSysTenantId()) { - result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); - } else { - for (TenantRepo repo : repos.values()) { - result += repo.countEntitiesByQuery(customerId, query, ignorePermissionCheck); - } - } + long result = get(tenantId).countEntitiesByQuery(customerId, query, ignorePermissionCheck); double timingMs = (double) (System.nanoTime() - startNs) / 1000_000; log.info("countEntitiesByQuery done in {} ms", timingMs); return result; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java index 44f8081759..970f8585dd 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/RepositoryUtils.java @@ -54,6 +54,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Optional; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -66,10 +67,10 @@ import static org.thingsboard.server.common.data.query.ComplexFilterPredicate.Co @Slf4j public class RepositoryUtils { - public static final Comparator SORT_ASC = Comparator.comparing(SortableEntityData::getSortValue) + public static final Comparator SORT_ASC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) .thenComparing(sp -> sp.getId().toString()); - public static final Comparator SORT_DESC = Comparator.comparing(SortableEntityData::getSortValue) + public static final Comparator SORT_DESC = Comparator.comparing((SortableEntityData sed) -> Optional.ofNullable(sed.getSortValue()).orElse("")) .thenComparing(sp -> sp.getId().toString()).reversed(); public static EntityType resolveEntityType(EntityFilter entityFilter) { @@ -206,11 +207,13 @@ public class RepositoryUtils { default -> throw new IllegalStateException(); }; } - DataPoint dp = entity.getDataPoint(keyFilter.key(), null); + DataKey dataKey = keyFilter.key(); + DataPoint dp = entity.getDataPoint(dataKey, null); boolean checkResult = switch (valueType) { case STRING -> { String str = dp != null ? dp.valueToString() : null; - yield StringUtils.isEmpty(str) || checkKeyFilter(str, keyFilter.predicate()); + yield (dataKey.type() == EntityKeyType.ENTITY_FIELD) ? (str == null || checkKeyFilter(str, keyFilter.predicate())) : + (str != null && checkKeyFilter(str, keyFilter.predicate())); } case BOOLEAN -> { Boolean booleanValue = dp != null ? dp.getBool() : null; diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 7f53829faf..777a77d054 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -132,7 +132,9 @@ public class BaseAttributesService implements AttributesService { for (TbPair keyVersionPair : result) { String key = keyVersionPair.getFirst(); Long version = keyVersionPair.getSecond(); - edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + if (version != null) { + edqsService.onDelete(tenantId, ObjectType.ATTRIBUTE_KV, new AttributeKv(entityId, scope, key, version)); + } keys.add(key); } return keys; diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index 4c1198721a..6e32901517 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -95,7 +95,7 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); - if (edqsApiService.isEnabled() && validForEdqs(query)) { + if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { EdqsRequest request = EdqsRequest.builder() .entityCountQuery(query) .build(); diff --git a/docker/.env b/docker/.env index 37c9768296..71722247df 100644 --- a/docker/.env +++ b/docker/.env @@ -14,6 +14,8 @@ COAP_TRANSPORT_DOCKER_NAME=tb-coap-transport LWM2M_TRANSPORT_DOCKER_NAME=tb-lwm2m-transport SNMP_TRANSPORT_DOCKER_NAME=tb-snmp-transport TB_VC_EXECUTOR_DOCKER_NAME=tb-vc-executor +EDQS_DOCKER_NAME=tb-edqs +EDQS_ENABLED=false TB_VERSION=latest diff --git a/docker/compose-utils.sh b/docker/compose-utils.sh index d5e60bee7f..5767026b11 100755 --- a/docker/compose-utils.sh +++ b/docker/compose-utils.sh @@ -128,6 +128,18 @@ function additionalStartupServices() { echo $ADDITIONAL_STARTUP_SERVICES } +function additionalComposeEdqsArgs() { + source .env + + if [ "$EDQS_ENABLED" = true ] + then + ADDITIONAL_COMPOSE_EDQS_ARGS="-f docker-compose.edqs.yml" + echo ADDITIONAL_COMPOSE_EDQS_ARGS + else + echo "" + fi +} + function permissionList() { PERMISSION_LIST=" 799 799 tb-node/log @@ -149,6 +161,12 @@ function permissionList() { " fi + if [ "$EDQS_ENABLED" = true ]; then + PERMISSION_LIST="$PERMISSION_LIST + 799 799 edqs/log + " + fi + CACHE="${CACHE:-redis}" case $CACHE in redis) diff --git a/docker/docker-compose.edqs.volumes.yml b/docker/docker-compose.edqs.volumes.yml new file mode 100644 index 0000000000..9d2ce946c8 --- /dev/null +++ b/docker/docker-compose.edqs.volumes.yml @@ -0,0 +1,30 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + tb-edqs-1: + volumes: + - tb-edqs-log-volume:/var/log/edqs + tb-edqs-2: + volumes: + - tb-edqs-log-volume:/var/log/edqs + +volumes: + tb-edqs-log-volume: + external: + name: ${TB_EDQS_LOG_VOLUME} diff --git a/docker/docker-compose.edqs.yml b/docker/docker-compose.edqs.yml new file mode 100644 index 0000000000..67e9c987e3 --- /dev/null +++ b/docker/docker-compose.edqs.yml @@ -0,0 +1,57 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '3.0' + +services: + tb-core1: + env_file: + - tb-core-edqs.env + tb-core2: + env_file: + - tb-core-edqs.env + tb-rule-engine1: + env_file: + - tb-rule-engine-edqs.env + tb-rule-engine2: + env_file: + - tb-rule-engine-edqs.env + tb-edqs-1: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + volumes: + - ./edqs/conf:/usr/share/edqs/conf + - ./edqs/log:/var/log/edqs + ports: + - "8080" + depends_on: + - zookeeper + - kafka + tb-edqs-2: + restart: always + image: "${DOCKER_REPO}/${EDQS_DOCKER_NAME}:${TB_VERSION}" + env_file: + - edqs.env + volumes: + - ./edqs/conf:/usr/share/edqs/conf + - ./edqs/log:/var/log/edqs + ports: + - "8080" + depends_on: + - zookeeper + - kafka diff --git a/docker/docker-install-tb.sh b/docker/docker-install-tb.sh index 1956e50eac..da09684d4f 100755 --- a/docker/docker-install-tb.sh +++ b/docker/docker-install-tb.sh @@ -49,6 +49,8 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? checkFolders --create || exit $? @@ -56,7 +58,8 @@ checkFolders --create || exit $? if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d ${ADDITIONAL_STARTUP_SERVICES}" case $COMPOSE_VERSION in @@ -73,7 +76,8 @@ if [ ! -z "${ADDITIONAL_STARTUP_SERVICES// }" ]; then fi COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=${loadDemo} \ tb-core1" diff --git a/docker/docker-remove-services.sh b/docker/docker-remove-services.sh index 6b36f4be08..3124119abc 100755 --- a/docker/docker-remove-services.sh +++ b/docker/docker-remove-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ down -v" case $COMPOSE_VERSION in diff --git a/docker/docker-start-services.sh b/docker/docker-start-services.sh index 3cdf10d00f..8b380f199b 100755 --- a/docker/docker-start-services.sh +++ b/docker/docker-start-services.sh @@ -29,10 +29,12 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + checkFolders --create || exit $? COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d" case $COMPOSE_VERSION in diff --git a/docker/docker-stop-services.sh b/docker/docker-stop-services.sh index 670c44ca92..54386d10dd 100755 --- a/docker/docker-stop-services.sh +++ b/docker/docker-stop-services.sh @@ -29,8 +29,10 @@ ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? ADDITIONAL_COMPOSE_MONITORING_ARGS=$(additionalComposeMonitoringArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} ${ADDITIONAL_COMPOSE_MONITORING_ARGS} ${ADDITIONAL_COMPOSE_EDQS_ARGS}\ stop" case $COMPOSE_VERSION in diff --git a/docker/docker-update-service.sh b/docker/docker-update-service.sh index 7a77241e48..de1fe0a89a 100755 --- a/docker/docker-update-service.sh +++ b/docker/docker-update-service.sh @@ -27,12 +27,16 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + COMPOSE_ARGS_PULL="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ pull" COMPOSE_ARGS_BUILD="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d --no-deps --build" case $COMPOSE_VERSION in diff --git a/docker/docker-upgrade-tb.sh b/docker/docker-upgrade-tb.sh index 3be5fbc14b..eca5d34957 100755 --- a/docker/docker-upgrade-tb.sh +++ b/docker/docker-upgrade-tb.sh @@ -42,21 +42,26 @@ ADDITIONAL_COMPOSE_ARGS=$(additionalComposeArgs) || exit $? ADDITIONAL_CACHE_ARGS=$(additionalComposeCacheArgs) || exit $? +ADDITIONAL_COMPOSE_EDQS_ARGS=$(additionalComposeEdqsArgs) || exit $? + ADDITIONAL_STARTUP_SERVICES=$(additionalStartupServices) || exit $? checkFolders --create || exit $? COMPOSE_ARGS_PULL="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ pull \ tb-core1" COMPOSE_ARGS_UP="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ up -d ${ADDITIONAL_STARTUP_SERVICES}" COMPOSE_ARGS_RUN="\ - -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} \ + -f docker-compose.yml ${ADDITIONAL_CACHE_ARGS} ${ADDITIONAL_COMPOSE_ARGS} ${ADDITIONAL_COMPOSE_QUEUE_ARGS} + ${ADDITIONAL_COMPOSE_EDQS_ARGS} \ run --no-deps --rm -e UPGRADE_TB=true -e FROM_VERSION=${fromVersion} \ tb-core1" diff --git a/docker/edqs.env b/docker/edqs.env new file mode 100644 index 0000000000..2c07d6e80d --- /dev/null +++ b/docker/edqs.env @@ -0,0 +1,7 @@ +ZOOKEEPER_ENABLED=true +ZOOKEEPER_URL=zookeeper:2181 +TB_KAFKA_SERVERS=kafka:9092 +HTTP_BIND_PORT=8080 + +METRICS_ENABLED=true +METRICS_ENDPOINTS_EXPOSE=prometheus diff --git a/docker/edqs/conf/edqs.conf b/docker/edqs/conf/edqs.conf new file mode 100644 index 0000000000..8c6b5d1826 --- /dev/null +++ b/docker/edqs/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=/var/log/edqs/${TB_SERVICE_ID}-gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=tb-edqs.out +export LOADER_PATH=/usr/share/edqs/conf diff --git a/docker/edqs/conf/logback.xml b/docker/edqs/conf/logback.xml new file mode 100644 index 0000000000..40481a8c35 --- /dev/null +++ b/docker/edqs/conf/logback.xml @@ -0,0 +1,52 @@ + + + + + + + /var/log/edqs/${TB_SERVICE_ID}/tb-edqs.log + + /var/log/edqs/tb-edqs.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + + + diff --git a/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json new file mode 100644 index 0000000000..3e913d4856 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/edqs_entities.json @@ -0,0 +1,161 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "iteration": 1737564772936, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "9BonzvTSz" + }, + "exemplar": true, + "expr": "sum by (objectType) (edqs_object_count{tenantId=~\"$tenantId\"})", + "interval": "", + "legendFormat": "{{objectType}}", + "refId": "A" + } + ], + "title": "EDQS object count", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "definition": "label_values(edqs_object_count, tenantId)", + "hide": 0, + "includeAll": true, + "label": "Tenant", + "multi": true, + "name": "tenantId", + "options": [], + "query": { + "query": "label_values(edqs_object_count, tenantId)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "EDQS", + "uid": "mK5A_DdHk", + "version": 9, + "weekStart": "" +} \ No newline at end of file diff --git a/docker/tb-core-edqs.env b/docker/tb-core-edqs.env new file mode 100644 index 0000000000..1417b780ec --- /dev/null +++ b/docker/tb-core-edqs.env @@ -0,0 +1,5 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_MODE=remote +TB_EDQS_SYNC_ENABLED=true +TB_EDQS_API_ENABLED=true \ No newline at end of file diff --git a/docker/tb-rule-engine-edqs.env b/docker/tb-rule-engine-edqs.env new file mode 100644 index 0000000000..82395ddcfe --- /dev/null +++ b/docker/tb-rule-engine-edqs.env @@ -0,0 +1,3 @@ +# ThingsBoard server configuration with enabled EDQS synchronization + +TB_EDQS_SYNC_ENABLED=true diff --git a/edqs/src/main/conf/edqs.conf b/edqs/src/main/conf/edqs.conf new file mode 100644 index 0000000000..3f96fd590d --- /dev/null +++ b/edqs/src/main/conf/edqs.conf @@ -0,0 +1,22 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +export JAVA_OPTS="$JAVA_OPTS -Xlog:gc*,heap*,age*,safepoint=debug:file=@pkg.logFolder@/gc.log:time,uptime,level,tags:filecount=10,filesize=10M" +export JAVA_OPTS="$JAVA_OPTS -XX:+IgnoreUnrecognizedVMOptions -XX:+HeapDumpOnOutOfMemoryError" +export JAVA_OPTS="$JAVA_OPTS -XX:-UseBiasedLocking -XX:+UseTLAB -XX:+ResizeTLAB -XX:+PerfDisableSharedMem -XX:+UseCondCardMark" +export JAVA_OPTS="$JAVA_OPTS -XX:+UseG1GC -XX:MaxGCPauseMillis=500 -XX:+UseStringDeduplication -XX:+ParallelRefProcEnabled -XX:MaxTenuringThreshold=10" +export LOG_FILENAME=${pkg.name}.out +export LOADER_PATH=${pkg.installFolder}/conf diff --git a/edqs/src/main/conf/logback.xml b/edqs/src/main/conf/logback.xml new file mode 100644 index 0000000000..850a28b212 --- /dev/null +++ b/edqs/src/main/conf/logback.xml @@ -0,0 +1,49 @@ + + + + + + + ${pkg.logFolder}/${pkg.name}.log + + ${pkg.logFolder}/${pkg.name}.%d{yyyy-MM-dd}.%i.log + 100MB + 30 + 3GB + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + %d{ISO8601} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 3c067ee641..65def6a964 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -51,6 +51,7 @@ public class ContainerTestSuite { private static final String TB_CORE_LOG_REGEXP = ".*Starting polling for events.*"; private static final String TRANSPORTS_LOG_REGEXP = ".*Going to recalculate partitions.*"; private static final String TB_VC_LOG_REGEXP = TRANSPORTS_LOG_REGEXP; + private static final String TB_EDQS_LOG_REGEXP = ".*All partitions processed.*"; private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); @@ -114,6 +115,8 @@ public class ContainerTestSuite { List composeFiles = new ArrayList<>(Arrays.asList( new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.edqs.yml"), + new File(targetDir + "docker-compose.edqs.volumes.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + "docker-compose.mosquitto.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), @@ -174,6 +177,8 @@ public class ContainerTestSuite { .withExposedService("broker", 1883) .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-rule-engine1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-rule-engine2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) @@ -182,7 +187,9 @@ public class ContainerTestSuite { .waitingFor("tb-lwm2m-transport", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-1", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-edqs-2", Wait.forLogMessage(TB_EDQS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); testContainer.start(); setActive(true); } catch (Exception e) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 94d0c64fa0..d74438a428 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityView; import org.thingsboard.server.common.data.EventInfo; import org.thingsboard.server.common.data.TbResource; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.asset.Asset; @@ -56,6 +57,9 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.page.TimePageLink; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; import org.thingsboard.server.common.data.rpc.Rpc; @@ -66,7 +70,6 @@ import org.thingsboard.server.common.data.security.DeviceCredentials; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import static io.restassured.RestAssured.given; import static java.net.HttpURLConnection.HTTP_BAD_REQUEST; @@ -110,6 +113,20 @@ public class TestRestClient { requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); } + public void resetToken() { + token = null; + refreshToken = null; + } + + public Tenant postTenant(Tenant tenant) { + return given().spec(requestSpec).body(tenant) + .post("/api/tenant") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Tenant.class); + } + public Device postDevice(String accessToken, Device device) { return given().spec(requestSpec).body(device) .pathParams("accessToken", accessToken) @@ -220,6 +237,15 @@ public class TestRestClient { .as(JsonNode.class); } + public JsonNode getLatestTelemetry(EntityId entityId) { + return given().spec(requestSpec) + .get("/api/plugins/telemetry/" + entityId.getEntityType().name() + "/" + entityId.getId() + "/values/timeseries") + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + public JsonPath postProvisionRequest(String provisionRequest) { return given().spec(requestSpec) .body(provisionRequest) @@ -479,6 +505,28 @@ public class TestRestClient { .as(User.class); } + public UserId createUserAndLogin(User user, String password) { + UserId userId = postUser(user).getId(); + getAndSetUserToken(userId); + return userId; + } + + public void getAndSetUserToken(UserId id) { + ObjectNode tokenInfo = given().spec(requestSpec) + .get("/api/user/" + id.getId().toString() + "/token") + .then() + .extract() + .as(ObjectNode.class); + token = tokenInfo.get("token").asText(); + refreshToken = tokenInfo.get("refreshToken").asText(); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + protected void resetTokens() { + this.token = null; + this.refreshToken = null; + } + public void deleteUser(UserId userId) { given().spec(requestSpec) .delete("/api/user/{userId}", userId.getId()) @@ -643,4 +691,45 @@ public class TestRestClient { } return urlParams; } + + public PageData postEntityDataQuery(EntityDataQuery entityDataQuery) { + return given().spec(requestSpec).body(entityDataQuery) + .post("/api/entitiesQuery/find") + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef<>() {}); + } + + public Long postCountDataQuery(EntityCountQuery entityCountQuery) { + return given().spec(requestSpec).body(entityCountQuery) + .post("/api/entitiesQuery/count") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Long.class); + } + + public Boolean isEdqsApiEnabled() { + return given().spec(requestSpec) + .get("/api/edqs/enabled") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Boolean.class); + } + + public void assignDeviceToCustomer(CustomerId customerId, DeviceId id) { + given().spec(requestSpec) + .post("/api/customer/" + customerId.getId().toString() + "/device/" + id.getId().toString()) + .then() + .statusCode(HTTP_OK); + } + + public void deleteTenant(TenantId tenantId) { + given().spec(requestSpec) + .delete("/api/tenant/" + tenantId.getId().toString()) + .then() + .statusCode(HTTP_OK); + } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index a71abe1781..9c1a90a4f1 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -48,6 +48,7 @@ public class ThingsBoardDbInstaller { private final static String TB_MQTT_TRANSPORT_LOG_VOLUME = "tb-mqtt-transport-log-test-volume"; private final static String TB_SNMP_TRANSPORT_LOG_VOLUME = "tb-snmp-transport-log-test-volume"; private final static String TB_VC_EXECUTOR_LOG_VOLUME = "tb-vc-executor-log-test-volume"; + private final static String TB_EDQS_LOG_VOLUME = "tb-edqs-log-test-volume"; private final static String JAVA_OPTS = "-Xmx512m"; private final DockerComposeExecutor dockerCompose; @@ -65,6 +66,7 @@ public class ThingsBoardDbInstaller { private final String tbMqttTransportLogVolume; private final String tbSnmpTransportLogVolume; private final String tbVcExecutorLogVolume; + private final String tbEdqsLogVolume; private final Map env; public ThingsBoardDbInstaller() { @@ -103,6 +105,7 @@ public class ThingsBoardDbInstaller { tbMqttTransportLogVolume = project + "_" + TB_MQTT_TRANSPORT_LOG_VOLUME; tbSnmpTransportLogVolume = project + "_" + TB_SNMP_TRANSPORT_LOG_VOLUME; tbVcExecutorLogVolume = project + "_" + TB_VC_EXECUTOR_LOG_VOLUME; + tbEdqsLogVolume = project + "_" + TB_EDQS_LOG_VOLUME; dockerCompose = new DockerComposeExecutor(composeFiles, project); @@ -119,6 +122,7 @@ public class ThingsBoardDbInstaller { env.put("TB_MQTT_TRANSPORT_LOG_VOLUME", tbMqttTransportLogVolume); env.put("TB_SNMP_TRANSPORT_LOG_VOLUME", tbSnmpTransportLogVolume); env.put("TB_VC_EXECUTOR_LOG_VOLUME", tbVcExecutorLogVolume); + env.put("TB_EDQS_LOG_VOLUME", tbEdqsLogVolume); if (IS_REDIS_CLUSTER) { for (int i = 0; i < 6; i++) { env.put("REDIS_CLUSTER_DATA_VOLUME_" + i, redisClusterDataVolume + '-' + i); @@ -189,6 +193,9 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("volume create " + tbVcExecutorLogVolume); dockerCompose.invokeDocker(); + dockerCompose.withCommand("volume create " + tbEdqsLogVolume); + dockerCompose.invokeDocker(); + StringBuilder additionalServices = new StringBuilder(); if (IS_HYBRID_MODE) { additionalServices.append(" cassandra"); @@ -220,7 +227,8 @@ public class ThingsBoardDbInstaller { dockerCompose.withCommand("up -d postgres" + additionalServices); dockerCompose.invokeCompose(); - dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true tb-core1"); + dockerCompose.withCommand("run --no-deps --rm -e INSTALL_TB=true -e LOAD_DEMO=true " + + "tb-core1"); dockerCompose.invokeCompose(); } finally { @@ -240,6 +248,7 @@ public class ThingsBoardDbInstaller { copyLogs(tbMqttTransportLogVolume, "./target/tb-mqtt-transport-logs/"); copyLogs(tbSnmpTransportLogVolume, "./target/tb-snmp-transport-logs/"); copyLogs(tbVcExecutorLogVolume, "./target/tb-vc-executor-logs/"); + copyLogs(tbEdqsLogVolume, "./target/tb-edqs-logs/"); StringJoiner rmVolumesCommand = new StringJoiner(" ") .add("volume rm -f") @@ -251,6 +260,7 @@ public class ThingsBoardDbInstaller { .add(tbMqttTransportLogVolume) .add(tbSnmpTransportLogVolume) .add(tbVcExecutorLogVolume) + .add(tbEdqsLogVolume) .add(resolveRedisComposeVolumeLog()); if (IS_HYBRID_MODE) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java new file mode 100644 index 0000000000..53d8a72e7f --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/edqs/EdqsEntityDataQueryTest.java @@ -0,0 +1,214 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.edqs; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.data.id.CustomerId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.query.DeviceTypeFilter; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityData; +import org.thingsboard.server.common.data.query.EntityDataPageLink; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.data.query.EntityDataSortOrder; +import org.thingsboard.server.common.data.query.EntityKey; +import org.thingsboard.server.common.data.query.EntityKeyType; +import org.thingsboard.server.common.data.query.EntityTypeFilter; +import org.thingsboard.server.common.data.query.TsValue; +import org.thingsboard.server.msa.AbstractContainerTest; +import org.thingsboard.server.msa.DisableUIListeners; +import org.thingsboard.server.msa.ui.utils.EntityPrototypes; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomer; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultCustomerAdmin; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultDeviceProfile; +import static org.thingsboard.server.msa.ui.utils.EntityPrototypes.defaultTenantAdmin; + +@DisableUIListeners +public class EdqsEntityDataQueryTest extends AbstractContainerTest { + + private TenantId tenantId; + private CustomerId customerId; + private TenantId tenantId2; + private CustomerId customerId2; + private UserId tenantAdminId; + private UserId customerUserId; + private UserId tenant2AdminId; + private UserId customer2UserId; + private final List tenantDevices = new ArrayList<>(); + private final List tenant2Devices = new ArrayList<>(); + private final String deviceProfile = "LoRa-" + RandomStringUtils.randomAlphabetic(10); + + @BeforeClass + public void beforeClass() throws Exception { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + await().atMost(60, TimeUnit.SECONDS).until(() -> testRestClient.isEdqsApiEnabled()); + + tenantId = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenantAdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId, "tenantAdmin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenantDevices, 97); + customerId = testRestClient.postCustomer(defaultCustomer(tenantId, "Customer")).getId(); + customerUserId = testRestClient.postUser(defaultCustomerAdmin(tenantId, customerId, "customerUser@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId, tenantDevices, 12); + + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + tenantId2 = testRestClient.postTenant(EntityPrototypes.defaultTenantPrototype("Tenant")).getId(); + tenant2AdminId = testRestClient.createUserAndLogin(defaultTenantAdmin(tenantId2, "tenant2Admin@thingsboard.org"), "tenant"); + testRestClient.postDeviceProfile(defaultDeviceProfile(deviceProfile)); + createDevices(deviceProfile, tenant2Devices, 97); + customerId2 = testRestClient.postCustomer(defaultCustomer(tenantId2, "Customer")).getId(); + customer2UserId = testRestClient.postUser(defaultCustomerAdmin(tenantId2, customerId2, "customer2User@thingsboard.org")).getId(); + assignDevicesToCustomer(customerId2, tenant2Devices, 12); + } + + @BeforeMethod + public void beforeMethod() { + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + } + + @AfterClass + public void afterClass() { + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.deleteTenant(tenantId); + testRestClient.deleteTenant(tenantId2); + } + + @Test + public void testSysAdminCountEntitiesByQuery() { + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery query = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).compareTo(97L * 2) >= 0); + + testRestClient.getAndSetUserToken(tenantAdminId); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getAndSetUserToken(tenant2AdminId); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(query).equals(97L)); + } + + @Test + public void testRetrieveTenantDevicesByDeviceTypeFilter() { + // login tenant admin + testRestClient.getAndSetUserToken(tenantAdminId); + checkUserDevices(tenantDevices); + + // login customer user + testRestClient.getAndSetUserToken(customerUserId); + checkUserDevices(tenantDevices.subList(0, 12)); + + // login other tenant admin + testRestClient.resetToken(); + testRestClient.login("sysadmin@thingsboard.org", "sysadmin"); + testRestClient.getAndSetUserToken(tenant2AdminId); + checkUserDevices(tenant2Devices); + } + + private void checkUserDevices(List devices) { + DeviceTypeFilter filter = new DeviceTypeFilter(); + filter.setDeviceTypes(List.of(deviceProfile)); + filter.setDeviceNameFilter(""); + + EntityDataSortOrder sortOrder = new EntityDataSortOrder(new EntityKey(EntityKeyType.ENTITY_FIELD, "createdTime"), EntityDataSortOrder.Direction.ASC); + EntityDataPageLink pageLink = new EntityDataPageLink(10, 0, null, sortOrder); + List entityFields = Collections.singletonList(new EntityKey(EntityKeyType.ENTITY_FIELD, "name")); + List latestFields = Collections.singletonList(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataQuery query = new EntityDataQuery(filter, pageLink, entityFields, latestFields, null); + + EntityTypeFilter allDeviceFilter = new EntityTypeFilter(); + allDeviceFilter.setEntityType(EntityType.DEVICE); + EntityCountQuery countQuery = new EntityCountQuery(allDeviceFilter); + await("Waiting for total device count") + .atMost(30, TimeUnit.SECONDS) + .until(() -> testRestClient.postCountDataQuery(countQuery).intValue() == devices.size()); + + PageData result = testRestClient.postEntityDataQuery(query); + assertThat(result.getTotalElements()).isEqualTo(devices.size()); + List retrievedDevices = result.getData(); + + assertThat(retrievedDevices).hasSize(10); + List retrievedDeviceNames = retrievedDevices.stream().map(entityData -> entityData.getLatest().get(EntityKeyType.ENTITY_FIELD).get("name").getValue()).toList(); + assertThat(retrievedDeviceNames).containsExactlyInAnyOrderElementsOf(devices.stream().map(Device::getName).toList().subList(0, 10)); + + //check temperature + for (int i = 0; i < 10; i++) { + Map> latest = retrievedDevices.get(i).getLatest(); + String name = latest.get(EntityKeyType.ENTITY_FIELD).get("name").getValue(); + assertThat(latest.get(EntityKeyType.TIME_SERIES).get("temperature").getValue()).isEqualTo(name.substring(name.length() - 1)); + } + } + + private String createDevices(String deviceType, List tenantDevices, int deviceCount) throws InterruptedException { + String prefix = StringUtils.randomAlphabetic(5); + for (int i = 0; i < deviceCount; i++) { + Device device = new Device(); + device.setName(prefix + "Device" + i); + device.setType(deviceType); + device.setLabel("testLabel" + (int) (Math.random() * 1000)); + //TO make sure devices have different created time + Thread.sleep(1); + String token = RandomStringUtils.randomAlphabetic(10); + Device saved = testRestClient.postDevice(token, device); + tenantDevices.add(saved); + + // save timeseries data + testRestClient.postTelemetry(token, createDeviceTelemetry(i)); + } + return deviceType; + } + + private void assignDevicesToCustomer(CustomerId customerId, List devices, int deviceCount) { + for (int i = 0; i < deviceCount; i++) { + Device device = devices.get(i); + testRestClient.assignDeviceToCustomer(customerId, device.getId()); + } + } + + protected ObjectNode createDeviceTelemetry(int temperature) { + ObjectNode objectNode = mapper.createObjectNode(); + objectNode.put("temperature", temperature); + return objectNode; + } + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java index ce5fee1f6f..91620271b7 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/utils/EntityPrototypes.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.DeviceProfileProvisionType; import org.thingsboard.server.common.data.DeviceProfileType; import org.thingsboard.server.common.data.DeviceTransportType; import org.thingsboard.server.common.data.EntityView; +import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmSeverity; @@ -37,12 +38,26 @@ import org.thingsboard.server.common.data.device.profile.DisabledDeviceProfilePr import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.security.Authority; public class EntityPrototypes { + public static Tenant defaultTenantPrototype(String tenantName) { + Tenant tenant = new Tenant(); + tenant.setTitle(tenantName); + return tenant; + } + + public static Customer defaultCustomer(TenantId tenantId, String title) { + Customer customer = new Customer(); + customer.setTenantId(tenantId); + customer.setTitle(title); + return customer; + } + public static Customer defaultCustomerPrototype(String entityName) { Customer customer = new Customer(); customer.setTitle(entityName); @@ -169,6 +184,23 @@ public class EntityPrototypes { return user; } + public static User defaultTenantAdmin(TenantId tenantId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setEmail(email); + user.setAuthority(Authority.TENANT_ADMIN); + return user; + } + + public static User defaultCustomerAdmin(TenantId tenantId, CustomerId customerId, String email) { + User user = new User(); + user.setTenantId(tenantId); + user.setCustomerId(customerId); + user.setEmail(email); + user.setAuthority(Authority.CUSTOMER_USER); + return user; + } + public static User defaultUser(String email, CustomerId customerId, String name) { User user = new User(); user.setEmail(email); diff --git a/msa/black-box-tests/src/test/resources/connectivity.xml b/msa/black-box-tests/src/test/resources/connectivity.xml index 2bde3f0a3f..425fbd67eb 100644 --- a/msa/black-box-tests/src/test/resources/connectivity.xml +++ b/msa/black-box-tests/src/test/resources/connectivity.xml @@ -22,6 +22,7 @@ + \ No newline at end of file diff --git a/msa/edqs/docker/Dockerfile b/msa/edqs/docker/Dockerfile new file mode 100644 index 0000000000..e9099c09c5 --- /dev/null +++ b/msa/edqs/docker/Dockerfile @@ -0,0 +1,31 @@ +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM thingsboard/openjdk17:bookworm-slim + +COPY start-tb-edqs.sh ${pkg.name}.deb /tmp/ + +RUN chmod a+x /tmp/*.sh \ + && mv /tmp/start-tb-edqs.sh /usr/bin && \ + (yes | dpkg -i /tmp/${pkg.name}.deb) && \ + rm /tmp/${pkg.name}.deb && \ + (systemctl --no-reload disable --now ${pkg.name}.service > /dev/null 2>&1 || :) && \ + chown -R ${pkg.user}:${pkg.user} /tmp && \ + chmod 555 ${pkg.installFolder}/bin/${pkg.name}.jar + +USER ${pkg.user} + +CMD ["start-tb-edqs.sh"] \ No newline at end of file diff --git a/msa/edqs/docker/start-tb-edqs.sh b/msa/edqs/docker/start-tb-edqs.sh new file mode 100755 index 0000000000..deb0f70eff --- /dev/null +++ b/msa/edqs/docker/start-tb-edqs.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Copyright © 2016-2025 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +CONF_FOLDER=${pkg.installFolder}/conf +jarfile=${pkg.installFolder}/bin/${pkg.name}.jar +configfile=${pkg.name}.conf + +source "${CONF_FOLDER}/${configfile}" + +echo "Starting '${project.name}' ..." + +cd ${pkg.installFolder}/bin + +exec java -cp ${jarfile} $JAVA_OPTS -Dloader.main=org.thingsboard.server.edqs.ThingsboardEdqsApplication \ + -Dspring.jpa.hibernate.ddl-auto=none \ + -Dlogging.config=$CONF_FOLDER/logback.xml \ + org.springframework.boot.loader.launch.PropertiesLauncher diff --git a/msa/edqs/pom.xml b/msa/edqs/pom.xml new file mode 100644 index 0000000000..f22cb0187c --- /dev/null +++ b/msa/edqs/pom.xml @@ -0,0 +1,190 @@ + + + 4.0.0 + + org.thingsboard + 4.0.0-SNAPSHOT + msa + + org.thingsboard.msa + edqs + pom + + ThingsBoard Entity Data Query Microservice + https://thingsboard.io + ThingsBoard Entity Data Query Microservice + + + UTF-8 + ${basedir}/../.. + edqs + tb-edqs + /var/log/${pkg.name} + /usr/share/${pkg.name} + pre-integration-test + + + + + org.thingsboard + edqs + ${project.version} + deb + deb + provided + + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-edqs + package + + copy + + + + + org.thingsboard + edqs + deb + deb + ${pkg.name}.deb + ${project.build.directory} + + + + + + + + org.apache.maven.plugins + maven-resources-plugin + + + copy-docker-config + process-resources + + copy-resources + + + ${project.build.directory} + + + docker + true + + + + + + + + com.spotify + dockerfile-maven-plugin + + + build-docker-image + pre-integration-test + + build + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + true + false + ${project.build.directory} + + + + tag-docker-image + pre-integration-test + + tag + + + ${dockerfile.skip} + ${docker.repo}/${docker.name} + ${project.version} + + + + + + + + + push-docker-image + + + push-docker-image + + + + + + com.spotify + dockerfile-maven-plugin + + + push-latest-docker-image + pre-integration-test + + push + + + latest + ${docker.repo}/${docker.name} + + + + push-version-docker-image + pre-integration-test + + push + + + ${project.version} + ${docker.repo}/${docker.name} + + + + + + + + + + + jenkins + Jenkins Repository + https://repo.jenkins-ci.org/releases + + false + + + + diff --git a/msa/pom.xml b/msa/pom.xml index 5ae0e903c1..98e820daaf 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -48,6 +48,7 @@ transport js-executor monitoring + edqs