From a5256c14e7076e7b93623f57ef7f8d85ba51e0bf Mon Sep 17 00:00:00 2001 From: Dmytro Skarzhynets Date: Mon, 1 Dec 2025 16:43:39 +0200 Subject: [PATCH] Fix broken rest client for `/entitiesQuery/find/keys`; refactoring --- .../controller/EntityQueryController.java | 86 ++++++------ .../query/DefaultEntityQueryService.java | 129 +++++------------- .../service/query/EntityQueryService.java | 9 +- .../dao/attributes/AttributesService.java | 2 +- .../dao/timeseries/TimeseriesService.java | 3 + .../data/query/AvailableEntityKeys.java | 67 +++++++++ .../server/dao/attributes/AttributesDao.java | 2 +- .../dao/attributes/BaseAttributesService.java | 7 +- .../attributes/CachedAttributesService.java | 6 +- .../dao/sql/attributes/JpaAttributeDao.java | 8 +- .../CachedRedisSqlTimeseriesLatestDao.java | 4 + .../dao/sqlts/SqlTimeseriesLatestDao.java | 12 +- .../dao/timeseries/BaseTimeseriesService.java | 9 +- .../CassandraBaseTimeseriesLatestDao.java | 8 +- .../dao/timeseries/TimeseriesLatestDao.java | 6 +- .../attributes/BaseAttributesServiceTest.java | 2 +- .../thingsboard/rest/client/RestClient.java | 20 ++- 17 files changed, 203 insertions(+), 177 deletions(-) create mode 100644 common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 70fa3a5ed0..f471c9e7b4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -17,33 +17,30 @@ package org.thingsboard.server.controller; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; +import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.edqs.EdqsState; import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilter; -import org.thingsboard.server.common.msg.edqs.EdqsApiService; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -51,52 +48,46 @@ import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.permission.Operation; import static org.thingsboard.server.controller.ControllerConstants.ALARM_DATA_QUERY_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.ATTRIBUTES_SCOPE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_COUNT_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_DATA_QUERY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; @RestController @TbCoreComponent @RequestMapping("/api") +@RequiredArgsConstructor public class EntityQueryController extends BaseController { - @Autowired - private EntityQueryService entityQueryService; - @Autowired - private EdqsService edqsService; - @Autowired - private EdqsApiService edqsApiService; + private final EntityQueryService entityQueryService; + private final EdqsService edqsService; private static final int MAX_PAGE_SIZE = 100; @ApiOperation(value = "Count Entities by Query", notes = ENTITY_COUNT_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/entitiesQuery/count", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/entitiesQuery/count") public long countEntitiesByQuery( @Parameter(description = "A JSON value representing the entity count query. See API call notes above for more details.") @RequestBody EntityCountQuery query) throws ThingsboardException { checkNotNull(query); resolveQuery(query); - return this.entityQueryService.countEntitiesByQuery(getCurrentUser(), query); + return entityQueryService.countEntitiesByQuery(getCurrentUser(), query); } @ApiOperation(value = "Find Entity Data by Query", notes = ENTITY_DATA_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/entitiesQuery/find", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/entitiesQuery/find") public PageData findEntityDataByQuery( @Parameter(description = "A JSON value representing the entity data query. See API call notes above for more details.") @RequestBody EntityDataQuery query) throws ThingsboardException { checkNotNull(query); resolveQuery(query); - return this.entityQueryService.findEntityDataByQuery(getCurrentUser(), query); + return entityQueryService.findEntityDataByQuery(getCurrentUser(), query); } @ApiOperation(value = "Find Alarms by Query", notes = ALARM_DATA_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/alarmsQuery/find", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/alarmsQuery/find") public PageData findAlarmDataByQuery( @Parameter(description = "A JSON value representing the alarm data query. See API call notes above for more details.") @RequestBody AlarmDataQuery query) throws ThingsboardException { @@ -107,13 +98,12 @@ public class EntityQueryController extends BaseController { checkUserId(assigneeId, Operation.READ); } resolveQuery(query); - return this.entityQueryService.findAlarmDataByQuery(getCurrentUser(), query); + return entityQueryService.findAlarmDataByQuery(getCurrentUser(), query); } @ApiOperation(value = "Count Alarms by Query (countAlarmsByQuery)", notes = "Returns the number of alarms that match the query definition.") @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/alarmsQuery/count", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/alarmsQuery/count") public long countAlarmsByQuery(@Parameter(description = "A JSON value representing the alarm count query.") @RequestBody AlarmCountQuery query) throws ThingsboardException { checkNotNull(query); @@ -122,31 +112,47 @@ public class EntityQueryController extends BaseController { checkUserId(assigneeId, Operation.READ); } resolveQuery(query); - return this.entityQueryService.countAlarmsByQuery(getCurrentUser(), query); + return entityQueryService.countAlarmsByQuery(getCurrentUser(), query); } - @ApiOperation(value = "Find Entity Keys by Query", - notes = "Uses entity data query (see 'Find Entity Data by Query') to find first 100 entities. Then fetch and return all unique time-series and/or attribute keys. Used mostly for UI hints.") + @ApiOperation( + value = "Find Available Entity Keys by Query", + notes = """ + Returns unique time series and/or attribute key names from entities matching the query.\n + Executes the Entity Data Query to find up to 100 entities, then fetches and aggregates all distinct key names.\n + Primarily used for UI features like autocomplete suggestions.""" + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH + ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/entitiesQuery/find/keys", method = RequestMethod.POST) - @ResponseBody - public DeferredResult findEntityTimeseriesAndAttributesKeysByQuery( - @Parameter(description = "A JSON value representing the entity data query. See API call notes above for more details.") + @PostMapping("/entitiesQuery/find/keys") + public DeferredResult findAvailableEntityKeysByQuery( + @Parameter(description = "Entity data query to find entities. Page size is capped at 100.") @RequestBody EntityDataQuery query, - @Parameter(description = "Include all unique time-series keys to the result.") - @RequestParam("timeseries") boolean isTimeseries, - @Parameter(description = "Include all unique attribute keys to the result.") - @RequestParam("attributes") boolean isAttributes, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) - @RequestParam(value = "scope", required = false) String scope) throws ThingsboardException { - TenantId tenantId = getTenantId(); - checkNotNull(query); + + // fixme: combination of timeseries = false and attributes = false is allowed, but always results in empty response, therefore does not make any sense + // such combinations should NOT be allowed, but changing this will break clients + + @Parameter(description = """ + When true, includes unique time series key names in the response. + When false, the 'timeseries' list will be empty.""") + @RequestParam("timeseries") boolean includeTimeseries, + + @Parameter(description = """ + When true, includes unique attribute key names in the response. + When false, the 'attribute' list will be empty. Use 'scope' parameter to filter by attribute scope.""") + @RequestParam("attributes") boolean includeAttributes, + + @Parameter(description = """ + Filters attribute keys by scope. Only applies when 'attributes' is true. + If not specified, returns attribute keys from all scopes.""", + schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) + @RequestParam(value = "scope", required = false) AttributeScope scope + ) throws ThingsboardException { resolveQuery(query); EntityDataPageLink pageLink = query.getPageLink(); if (pageLink.getPageSize() > MAX_PAGE_SIZE) { pageLink.setPageSize(MAX_PAGE_SIZE); } - return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope); + return wrapFuture(entityQueryService.getKeysByQuery(getCurrentUser(), getTenantId(), query, includeTimeseries, includeAttributes, scope)); } @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index f39769526a..6ef6b239e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -15,24 +15,18 @@ */ package org.thingsboard.server.service.query; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -40,6 +34,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.EntityCountQuery; @@ -56,16 +51,13 @@ import org.thingsboard.server.common.data.query.SimpleKeyFilterPredicate; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; -import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -73,9 +65,10 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; import java.util.stream.Collectors; +import static com.google.common.util.concurrent.Futures.immediateFuture; + @Service @Slf4j @TbCoreComponent @@ -138,20 +131,12 @@ public class DefaultEntityQueryService implements EntityQueryService { } private void resolveDynamicValue(DynamicValue dynamicValue, SecurityUser user, FilterPredicateType predicateType) { - EntityId entityId; - switch (dynamicValue.getSourceType()) { - case CURRENT_TENANT: - entityId = user.getTenantId(); - break; - case CURRENT_CUSTOMER: - entityId = user.getCustomerId(); - break; - case CURRENT_USER: - entityId = user.getId(); - break; - default: - throw new RuntimeException("Not supported operation for source type: {" + dynamicValue.getSourceType() + "}"); - } + EntityId entityId = switch (dynamicValue.getSourceType()) { + case CURRENT_TENANT -> user.getTenantId(); + case CURRENT_CUSTOMER -> user.getCustomerId(); + case CURRENT_USER -> user.getId(); + default -> throw new RuntimeException("Not supported operation for source type: {" + dynamicValue.getSourceType() + "}"); + }; try { Optional valueOpt = attributesService.find(user.getTenantId(), entityId, @@ -242,101 +227,51 @@ public class DefaultEntityQueryService implements EntityQueryService { } @Override - public DeferredResult getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, - boolean isTimeseries, boolean isAttributes, String attributesScope) { - final DeferredResult response = new DeferredResult<>(); + public ListenableFuture getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, + boolean isTimeseries, boolean isAttributes, AttributeScope scope) { if (!isAttributes && !isTimeseries) { - replyWithEmptyResponse(response); - return response; + return immediateFuture(AvailableEntityKeys.none()); } - List ids = this.findEntityDataByQuery(securityUser, query).getData().stream() + List ids = findEntityDataByQuery(securityUser, query).getData().stream() .map(EntityData::getEntityId) - .collect(Collectors.toList()); + .toList(); if (ids.isEmpty()) { - replyWithEmptyResponse(response); - return response; + return immediateFuture(AvailableEntityKeys.none()); } Set types = ids.stream().map(EntityId::getEntityType).collect(Collectors.toSet()); - final ListenableFuture> timeseriesKeysFuture; - final ListenableFuture> attributesKeysFuture; + ListenableFuture> timeseriesKeysFuture; + ListenableFuture> attributesKeysFuture; if (isTimeseries) { - timeseriesKeysFuture = dbCallbackExecutor.submit(() -> timeseriesService.findAllKeysByEntityIds(tenantId, ids)); + timeseriesKeysFuture = timeseriesService.findAllKeysByEntityIdsAsync(tenantId, ids); } else { - timeseriesKeysFuture = null; + timeseriesKeysFuture = immediateFuture(Collections.emptyList()); } if (isAttributes) { Map> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType)); List>> futures = new ArrayList<>(typesMap.size()); - typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, entityIds, attributesScope)))); + typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, entityIds, scope)))); attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> { if (CollectionUtils.isEmpty(lists)) { return Collections.emptyList(); } - return lists.stream().flatMap(List::stream).distinct().sorted().collect(Collectors.toList()); - }, dbCallbackExecutor); - } else { - attributesKeysFuture = null; - } - - if (isTimeseries && isAttributes) { - Futures.whenAllComplete(timeseriesKeysFuture, attributesKeysFuture).run(() -> { - try { - replyWithResponse(response, types, timeseriesKeysFuture.get(), attributesKeysFuture.get()); - } catch (Exception e) { - log.error("Failed to fetch timeseries and attributes keys!", e); - AccessValidator.handleError(e, response, HttpStatus.INTERNAL_SERVER_ERROR); - } + return lists.stream().flatMap(List::stream).distinct().sorted().toList(); }, dbCallbackExecutor); - } else if (isTimeseries) { - addCallback(timeseriesKeysFuture, keys -> replyWithResponse(response, types, keys, null), - error -> { - log.error("Failed to fetch timeseries keys!", error); - AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR); - }); } else { - addCallback(attributesKeysFuture, keys -> replyWithResponse(response, types, null, keys), - error -> { - log.error("Failed to fetch attributes keys!", error); - AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR); - }); + attributesKeysFuture = immediateFuture(Collections.emptyList()); } - return response; - } - - private void replyWithResponse(DeferredResult response, Set types, List timeseriesKeys, List attributesKeys) { - ObjectNode json = JacksonUtil.newObjectNode(); - addItemsToArrayNode(json.putArray("entityTypes"), types); - addItemsToArrayNode(json.putArray("timeseries"), timeseriesKeys); - addItemsToArrayNode(json.putArray("attribute"), attributesKeys); - response.setResult(new ResponseEntity<>(json, HttpStatus.OK)); - } - private void replyWithEmptyResponse(DeferredResult response) { - replyWithResponse(response, Collections.emptySet(), Collections.emptyList(), Collections.emptyList()); - } - - private void addItemsToArrayNode(ArrayNode arrayNode, Collection collection) { - if (!CollectionUtils.isEmpty(collection)) { - collection.forEach(item -> arrayNode.add(item.toString())); - } - } - - private void addCallback(ListenableFuture> future, Consumer> success, Consumer error) { - Futures.addCallback(future, new FutureCallback>() { - @Override - public void onSuccess(@Nullable List keys) { - success.accept(keys); - } - - @Override - public void onFailure(Throwable t) { - error.accept(t); - } - }, dbCallbackExecutor); + return Futures.whenAllComplete(timeseriesKeysFuture, attributesKeysFuture) + .call(() -> { + try { + return new AvailableEntityKeys(types, Futures.getDone(timeseriesKeysFuture), Futures.getDone(attributesKeysFuture)); + } catch (ExecutionException e) { + throw new ThingsboardException(e.getCause(), ThingsboardErrorCode.DATABASE); + } + }, dbCallbackExecutor); } } diff --git a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java index 78ea2519fd..ac6553d738 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.service.query; -import org.springframework.http.ResponseEntity; -import org.springframework.web.context.request.async.DeferredResult; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -37,7 +38,7 @@ public interface EntityQueryService { long countAlarmsByQuery(SecurityUser securityUser, AlarmCountQuery query); - DeferredResult getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, - boolean isTimeseries, boolean isAttributes, String attributesScope); + ListenableFuture getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, + boolean isTimeseries, boolean isAttributes, AttributeScope scope); } diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 0d5d3dcd13..2cdad499de 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -48,7 +48,7 @@ public interface AttributesService { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); - List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope); + List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope); int removeAllByEntityId(TenantId tenantId, EntityId entityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index e239e22ee9..0b88ce17cc 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -63,5 +63,8 @@ public interface TimeseriesService { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds); + void cleanup(long systemTtl); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java new file mode 100644 index 0000000000..4cac646908 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.List; +import java.util.Set; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Objects.requireNonNullElse; + +@Schema( + description = "Contains unique time series and attribute key names discovered from entities matching a query. Used primarily for UI hints such as autocomplete suggestions." +) +public record AvailableEntityKeys( + @Schema( + description = "Set of entity types found among the matched entities.", + example = "[\"DEVICE\", \"ASSET\"]", + requiredMode = Schema.RequiredMode.REQUIRED + ) + Set entityTypes, + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @ArraySchema( + arraySchema = @Schema(description = "List of unique time series key names available on the matched entities."), + schema = @Schema(implementation = String.class, example = "temperature"), + uniqueItems = true + ) + List timeseries, + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @ArraySchema( + arraySchema = @Schema(description = "List of unique attribute key names available on the matched entities."), + schema = @Schema(implementation = String.class, example = "serialNumber"), + uniqueItems = true + ) + List attribute +) { + + public AvailableEntityKeys { + entityTypes = requireNonNullElse(entityTypes, emptySet()); + timeseries = requireNonNullElse(timeseries, emptyList()); + attribute = requireNonNullElse(attribute, emptyList()); + } + + public static AvailableEntityKeys none() { + return new AvailableEntityKeys(emptySet(), emptyList(), emptyList()); + } + +} diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 5527d17add..a1af985887 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -53,7 +53,7 @@ public interface AttributesDao { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); - List findAllKeysByEntityIdsAndAttributeType(TenantId tenantId, List entityIds, String attributeType); + List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope); List> removeAllByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 62b35caa7f..362aa95ee6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -28,7 +28,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -93,11 +92,11 @@ public class BaseAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope) { - if (StringUtils.isEmpty(scope)) { + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope) { + if (scope == null) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } else { - return attributesDao.findAllKeysByEntityIdsAndAttributeType(tenantId, entityIds, scope); + return attributesDao.findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 44b2daaf8e..11ba48773d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -212,11 +212,11 @@ public class CachedAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope) { - if (StringUtils.isEmpty(scope)) { + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope) { + if (scope == null) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } else { - return attributesDao.findAllKeysByEntityIdsAndAttributeType(tenantId, entityIds, scope); + return attributesDao.findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 0a8b8f6399..1e58b23a1f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -177,10 +177,12 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public List findAllKeysByEntityIdsAndAttributeType(TenantId tenantId, List entityIds, String attributeType) { + public List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope) { return attributeKvRepository - .findAllKeysByEntityIdsAndAttributeType(entityIds.stream().map(EntityId::getId).collect(Collectors.toList()), AttributeScope.valueOf(attributeType).getId()) - .stream().map(id -> keyDictionaryDao.getKey(id)).collect(Collectors.toList()); + .findAllKeysByEntityIdsAndAttributeType(entityIds.stream().map(EntityId::getId).toList(), scope.getId()) + .stream() + .map(keyDictionaryDao::getKey) + .toList(); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index d45182442a..bac5529249 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -167,5 +167,9 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return sqlDao.findAllKeysByEntityIdsAsync(tenantId, entityIds); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index c546fc21ea..27470bfe8a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -24,7 +24,6 @@ import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -38,8 +37,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; @@ -64,7 +61,6 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.function.Function; -import java.util.stream.Collectors; @Slf4j @Component @@ -185,9 +181,13 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { - return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).toList()); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return service.submit(() -> findAllKeysByEntityIds(tenantId, entityIds)); + } private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query, Long version) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); @@ -211,7 +211,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme ReadTsKvQueryResult::getData, MoreExecutors.directExecutor()); } - protected TsKvEntry doFindLatestSync(EntityId entityId, String key) { + protected TsKvEntry doFindLatestSync(EntityId entityId, String key) { TsKvLatestCompositeKey compositeKey = new TsKvLatestCompositeKey( entityId.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index ceb7fcf822..de197bf88f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -156,6 +156,11 @@ public class BaseTimeseriesService implements TimeseriesService { return timeseriesLatestDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return timeseriesLatestDao.findAllKeysByEntityIdsAsync(tenantId, entityIds); + } + @Override public void cleanup(long systemTtl) { timeseriesDao.cleanup(systemTtl); @@ -300,13 +305,13 @@ public class BaseTimeseriesService implements TimeseriesService { long interval = query.getInterval(); if (interval < 1) { throw new IncorrectParameterException("Invalid TsKvQuery: 'interval' must be greater than 0, but got " + interval + - ". Please check your query parameters and ensure 'endTs' is greater than 'startTs' or increase 'interval'."); + ". Please check your query parameters and ensure 'endTs' is greater than 'startTs' or increase 'interval'."); } long step = Math.max(interval, 1000); long intervalCounts = (query.getEndTs() - query.getStartTs()) / step; if (intervalCounts > maxTsIntervals || intervalCounts < 0) { throw new IncorrectParameterException("Incorrect TsKvQuery. Number of intervals is to high - " + intervalCounts + ". " + - "Please increase 'interval' parameter for your query or reduce the time range of the query."); + "Please increase 'interval' parameter for your query or reduce the time range of the query."); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 54a7e68725..dd44a62349 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -36,17 +36,13 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -103,6 +99,10 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return Futures.immediateFuture(Collections.emptyList()); + } @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 32479301ae..74c041e4d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -22,12 +22,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import java.util.List; -import java.util.Map; import java.util.Optional; public interface TimeseriesLatestDao { @@ -54,4 +50,6 @@ public interface TimeseriesLatestDao { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds); + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index 5978d903c7..41a013e4c9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -223,7 +223,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { saveAttribute(tenantId, deviceId, AttributeScope.SERVER_SCOPE, "key2", "123"); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - List keys = attributesService.findAllKeysByEntityIds(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE.name()); + List keys = attributesService.findAllKeysByEntityIds(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE); assertThat(keys).containsOnly("key1", "key2"); }); } diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 2c8e01d246..6eb61b3e13 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -39,10 +39,12 @@ import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rest.client.utils.RestJsonConverter; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -160,6 +162,7 @@ import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -592,7 +595,7 @@ public class RestClient implements Closeable { } public PageData getAllAlarmsV2(List statusList, List severityList, - List typeList, String assignedId, TimePageLink pageLink) { + List typeList, String assignedId, TimePageLink pageLink) { String urlSecondPart = "/api/v2/alarms?"; Map params = new HashMap<>(); if (!CollectionUtils.isEmpty(statusList)) { @@ -1824,12 +1827,15 @@ public class RestClient implements Closeable { }).getBody(); } - public JsonNode findEntityTimeseriesAndAttributesKeysByQuery(EntityDataQuery query) { - return restTemplate.exchange( - baseURL + "/api/entitiesQuery/find/keys", - HttpMethod.POST, new HttpEntity<>(query), - new ParameterizedTypeReference() { - }).getBody(); + public AvailableEntityKeys findAvailableEntityKeysByQuery(EntityDataQuery query, boolean includeTimeseries, boolean includeAttributes, AttributeScope scope) { + var uri = UriComponentsBuilder.fromUriString(baseURL) + .path("/api/entitiesQuery/find/keys") + .queryParam("timeseries", includeTimeseries) + .queryParam("attributes", includeAttributes) + .queryParamIfPresent("scope", Optional.ofNullable(scope)) + .build() + .toUri(); + return restTemplate.exchange(uri, HttpMethod.POST, new HttpEntity<>(query), new ParameterizedTypeReference() {}).getBody(); } public PageData findAlarmDataByQuery(AlarmDataQuery query) {