diff --git a/application/pom.xml b/application/pom.xml index 4cbd9c3b64..86d50604e7 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -382,35 +382,35 @@ rocksdbjni - dev.langchain4j + org.thingsboard.langchain4j langchain4j-open-ai - dev.langchain4j + org.thingsboard.langchain4j langchain4j-azure-open-ai - dev.langchain4j + org.thingsboard.langchain4j langchain4j-google-ai-gemini - dev.langchain4j + org.thingsboard.langchain4j langchain4j-vertex-ai-gemini - dev.langchain4j + org.thingsboard.langchain4j langchain4j-mistral-ai - dev.langchain4j + org.thingsboard.langchain4j langchain4j-anthropic - dev.langchain4j + org.thingsboard.langchain4j langchain4j-bedrock - dev.langchain4j + org.thingsboard.langchain4j langchain4j-github-models @@ -420,7 +420,7 @@ - dev.langchain4j + org.thingsboard.langchain4j langchain4j-ollama diff --git a/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java b/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java index 410b05456e..f6851a5a71 100644 --- a/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/SwaggerConfiguration.java @@ -86,6 +86,9 @@ public class SwaggerConfiguration { public static final String LOGIN_ENDPOINT = "/api/auth/login"; public static final String REFRESH_TOKEN_ENDPOINT = "/api/auth/token"; + private static final String LOGIN_PASSWORD_SCHEME = "HTTP login form"; + private static final String API_KEY_SCHEME = "API key form"; + private static final ApiResponses loginResponses = loginResponses(); private static final ApiResponses defaultErrorResponses = defaultErrorResponses(false); private static final ApiResponses defaultPostErrorResponses = defaultErrorResponses(true); @@ -142,14 +145,28 @@ public class SwaggerConfiguration { .license(license) .version(apiVersion); - SecurityScheme securityScheme = new SecurityScheme() + SecurityScheme loginPasswordScheme = new SecurityScheme() .type(SecurityScheme.Type.HTTP) .description("Enter Username / Password") .scheme("loginPassword") .bearerFormat("/api/auth/login|X-Authorization"); + SecurityScheme apiKeyScheme = new SecurityScheme() + .type(SecurityScheme.Type.APIKEY) + .name("X-Authorization") + .in(SecurityScheme.In.HEADER) + .description(""" + Enter the API key value with 'ApiKey' prefix in format: **ApiKey ** + + Example: **ApiKey tb_5te51SkLRYpjGrujUGwqkjFvooWBlQpVe2An2Dr3w13wjfxDW** + +
**NOTE**: Use only ONE authentication method at a time. If both are authorized, JWT auth takes the priority.
+ """); + var openApi = new OpenAPI() - .components(new Components().addSecuritySchemes("HTTP login form", securityScheme)) + .components(new Components() + .addSecuritySchemes(LOGIN_PASSWORD_SCHEME, loginPasswordScheme) + .addSecuritySchemes(API_KEY_SCHEME, apiKeyScheme)) .info(info); addDefaultSchemas(openApi); addLoginOperation(openApi); @@ -198,13 +215,14 @@ public class SwaggerConfiguration { operation.summary("Login method to get user JWT token data"); operation.description(""" Login method used to authenticate user and get JWT token data. - + Value of the response **token** field can be used as **X-Authorization** header value: - + `X-Authorization: Bearer $JWT_TOKEN_VALUE`."""); + var requestBody = new RequestBody().description("Login request") .content(new Content().addMediaType(APPLICATION_JSON_VALUE, - new MediaType().schema(new Schema().$ref("#/components/schemas/LoginRequest")))); + new MediaType().schema(new Schema().$ref("#/components/schemas/LoginRequest")))); operation.requestBody(requestBody); operation.responses(loginResponses); @@ -218,11 +236,11 @@ public class SwaggerConfiguration { var operation = new Operation(); operation.summary("Refresh user JWT token data"); operation.description(""" - Method to refresh JWT token. Provide a valid refresh token to get a new JWT token. - - The response contains a new token that can be used for authorization. - - `X-Authorization: Bearer $JWT_TOKEN_VALUE`"""); + Method to refresh JWT token. Provide a valid refresh token to get a new JWT token. + + The response contains a new token that can be used for authorization. + + `X-Authorization: Bearer $JWT_TOKEN_VALUE`"""); var requestBody = new RequestBody().description("Refresh token request") .content(new Content().addMediaType(APPLICATION_JSON_VALUE, @@ -291,8 +309,9 @@ public class SwaggerConfiguration { return (routerOperation, handlerMethod) -> { String[] pNames = localSpringDocParameterNameDiscoverer.getParameterNames(handlerMethod.getMethod()); String[] reflectionParametersNames = Arrays.stream(handlerMethod.getMethod().getParameters()).map(java.lang.reflect.Parameter::getName).toArray(String[]::new); - if (pNames == null || Arrays.stream(pNames).anyMatch(Objects::isNull)) + if (pNames == null || Arrays.stream(pNames).anyMatch(Objects::isNull)) { pNames = reflectionParametersNames; + } MethodParameter[] parameters = handlerMethod.getMethodParameters(); List requestParams = new ArrayList<>(); for (var i = 0; i < parameters.length; i++) { @@ -324,26 +343,25 @@ public class SwaggerConfiguration { } private OpenApiCustomizer customOpenApiCustomizer() { - var loginForm = new SecurityRequirement().addList("HTTP login form", Arrays.asList( - Authority.SYS_ADMIN.name(), - Authority.TENANT_ADMIN.name(), - Authority.CUSTOMER_USER.name() - )); + var loginRequirement = createSecurityRequirement(LOGIN_PASSWORD_SCHEME); + var apiKeyRequirement = createSecurityRequirement(API_KEY_SCHEME); + return openAPI -> { var paths = openAPI.getPaths(); - paths.entrySet().stream().peek(entry -> { - securityCustomization(loginForm, entry); - if (!entry.getKey().equals(LOGIN_ENDPOINT)) { - defaultErrorResponsesCustomization(entry.getValue()); - } - }).map(this::tagsCustomization).filter(Objects::nonNull).distinct().sorted(Comparator.comparing(Tag::getName)).forEach(openAPI::addTagsItem); + paths.entrySet().stream() + .peek(entry -> { + securityCustomization(entry, loginRequirement, apiKeyRequirement); + if (!entry.getKey().equals(LOGIN_ENDPOINT)) { + defaultErrorResponsesCustomization(entry.getValue()); + } + }) + .map(this::extractTagFromPath).filter(Objects::nonNull).distinct().sorted(Comparator.comparing(Tag::getName)).forEach(openAPI::addTagsItem); var pathItemsByTags = new TreeMap>(); paths.forEach((k, v) -> { var tagItem = tagItemFromPathItem(v); if (tagItem != null) { - var pathItemMap = pathItemsByTags.computeIfAbsent(tagItem, k1 -> new TreeMap<>()); - pathItemMap.put(k, v); + pathItemsByTags.computeIfAbsent(tagItem, k1 -> new TreeMap<>()).put(k, v); } }); var sortedPaths = new Paths(); @@ -357,13 +375,17 @@ public class SwaggerConfiguration { }; } + private SecurityRequirement createSecurityRequirement(String schemeName) { + return new SecurityRequirement().addList(schemeName, Arrays.asList( + Authority.SYS_ADMIN.name(), + Authority.TENANT_ADMIN.name(), + Authority.CUSTOMER_USER.name() + )); + } - private Tag tagsCustomization(Map.Entry entry) { - var tagItem = tagItemFromPathItem(entry.getValue()); - if (tagItem != null) { - return tagFromTagItem(tagItem); - } - return null; + private Tag extractTagFromPath(Map.Entry entry) { + var tagName = tagItemFromPathItem(entry.getValue()); + return tagName != null ? tagFromTagItem(tagName) : null; } private String tagItemFromPathItem(PathItem item) { @@ -383,17 +405,20 @@ public class SwaggerConfiguration { StringBuilder sb = new StringBuilder(); for (String word : words) { - sb.append(word.substring(0, 1).toUpperCase()); - sb.append(word.substring(1).toLowerCase()); - sb.append(" "); + if (!word.isEmpty()) { + sb.append(word.substring(0, 1).toUpperCase()); + sb.append(word.substring(1).toLowerCase()); + sb.append(" "); + } } return new Tag().name(tagItem).description(sb.toString().trim()); } private void defaultErrorResponsesCustomization(PathItem pathItem) { - pathItem.readOperationsMap().forEach(((httpMethod, operation) -> { + pathItem.readOperationsMap().forEach((httpMethod, operation) -> { var errorResponses = httpMethod.equals(PathItem.HttpMethod.POST) ? defaultPostErrorResponses : defaultErrorResponses; + var responses = operation.getResponses(); if (responses == null) { responses = errorResponses; @@ -406,16 +431,19 @@ public class SwaggerConfiguration { }); } operation.setResponses(responses); - })); + }); } - private void securityCustomization(SecurityRequirement loginForm, Map.Entry entry) { + private void securityCustomization(Map.Entry entry, SecurityRequirement jwtBearerRequirement, SecurityRequirement apiKeyRequirement) { var path = entry.getKey(); - if (path.matches(securityPathRegex) && !path.matches(nonSecurityPathRegex) && !path.equals(LOGIN_ENDPOINT)) { + if (path.matches(securityPathRegex) && !path.matches(nonSecurityPathRegex) && !path.equals(LOGIN_ENDPOINT) && !path.equals(REFRESH_TOKEN_ENDPOINT)) { entry.getValue() .readOperationsMap() .values() - .forEach(operation -> operation.addSecurityItem(loginForm)); + .forEach(operation -> { + operation.addSecurityItem(jwtBearerRequirement); + operation.addSecurityItem(apiKeyRequirement); + }); } } @@ -430,6 +458,7 @@ public class SwaggerConfiguration { private static ApiResponses defaultErrorResponses(boolean isPost) { ApiResponses apiResponses = new ApiResponses(); + apiResponses.addApiResponse("400", errorResponse("400", "Bad Request", ThingsboardErrorResponse.of(isPost ? "Invalid request body" : "Invalid UUID string: 123", ThingsboardErrorCode.BAD_REQUEST_PARAMS, HttpStatus.BAD_REQUEST))); @@ -465,8 +494,7 @@ public class SwaggerConfiguration { ThingsboardErrorResponse.of("Authentication failed", ThingsboardErrorCode.AUTHENTICATION, HttpStatus.UNAUTHORIZED)) ) )); - var credentialsExpiredSchema = new Schema(); - credentialsExpiredSchema.$ref("#/components/schemas/ThingsboardCredentialsExpiredResponse"); + var credentialsExpiredSchema = new Schema().$ref("#/components/schemas/ThingsboardCredentialsExpiredResponse"); apiResponses.addApiResponse("401 ", errorResponse("Unauthorized (**Expired credentials**)", Map.of( "credentials-expired", errorExample("Expired credentials", @@ -482,15 +510,13 @@ public class SwaggerConfiguration { } private static ApiResponse errorResponse(String description, Map examples) { - var schema = new Schema(); - schema.$ref("#/components/schemas/ThingsboardErrorResponse"); + var schema = new Schema().$ref("#/components/schemas/ThingsboardErrorResponse"); return errorResponse(description, examples, schema); } private static ApiResponse errorResponse(String description, Map examples, Schema errorResponseSchema) { - MediaType mediaType = new MediaType().schema(errorResponseSchema); - mediaType.setExamples(examples); - Content content = new Content().addMediaType(org.springframework.http.MediaType.APPLICATION_JSON_VALUE, mediaType); + MediaType mediaType = new MediaType().schema(errorResponseSchema).examples(examples); + Content content = new Content().addMediaType(org.springframework.http.MediaType.APPLICATION_JSON_VALUE, mediaType); return new ApiResponse().description(description).content(content); } diff --git a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java index 70fa3a5ed0..f471c9e7b4 100644 --- a/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java +++ b/application/src/main/java/org/thingsboard/server/controller/EntityQueryController.java @@ -17,33 +17,30 @@ package org.thingsboard.server.controller; import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.media.Schema; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.ResponseEntity; +import lombok.RequiredArgsConstructor; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.async.DeferredResult; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.edqs.EdqsState; import org.thingsboard.server.common.data.edqs.ToCoreEdqsRequest; import org.thingsboard.server.common.data.exception.ThingsboardException; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataPageLink; import org.thingsboard.server.common.data.query.EntityDataQuery; import org.thingsboard.server.common.data.query.EntityFilter; -import org.thingsboard.server.common.msg.edqs.EdqsApiService; import org.thingsboard.server.common.msg.edqs.EdqsService; import org.thingsboard.server.config.annotations.ApiOperation; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -51,52 +48,46 @@ import org.thingsboard.server.service.query.EntityQueryService; import org.thingsboard.server.service.security.permission.Operation; import static org.thingsboard.server.controller.ControllerConstants.ALARM_DATA_QUERY_DESCRIPTION; -import static org.thingsboard.server.controller.ControllerConstants.ATTRIBUTES_SCOPE_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_COUNT_QUERY_DESCRIPTION; import static org.thingsboard.server.controller.ControllerConstants.ENTITY_DATA_QUERY_DESCRIPTION; +import static org.thingsboard.server.controller.ControllerConstants.TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH; @RestController @TbCoreComponent @RequestMapping("/api") +@RequiredArgsConstructor public class EntityQueryController extends BaseController { - @Autowired - private EntityQueryService entityQueryService; - @Autowired - private EdqsService edqsService; - @Autowired - private EdqsApiService edqsApiService; + private final EntityQueryService entityQueryService; + private final EdqsService edqsService; private static final int MAX_PAGE_SIZE = 100; @ApiOperation(value = "Count Entities by Query", notes = ENTITY_COUNT_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/entitiesQuery/count", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/entitiesQuery/count") public long countEntitiesByQuery( @Parameter(description = "A JSON value representing the entity count query. See API call notes above for more details.") @RequestBody EntityCountQuery query) throws ThingsboardException { checkNotNull(query); resolveQuery(query); - return this.entityQueryService.countEntitiesByQuery(getCurrentUser(), query); + return entityQueryService.countEntitiesByQuery(getCurrentUser(), query); } @ApiOperation(value = "Find Entity Data by Query", notes = ENTITY_DATA_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/entitiesQuery/find", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/entitiesQuery/find") public PageData findEntityDataByQuery( @Parameter(description = "A JSON value representing the entity data query. See API call notes above for more details.") @RequestBody EntityDataQuery query) throws ThingsboardException { checkNotNull(query); resolveQuery(query); - return this.entityQueryService.findEntityDataByQuery(getCurrentUser(), query); + return entityQueryService.findEntityDataByQuery(getCurrentUser(), query); } @ApiOperation(value = "Find Alarms by Query", notes = ALARM_DATA_QUERY_DESCRIPTION) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/alarmsQuery/find", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/alarmsQuery/find") public PageData findAlarmDataByQuery( @Parameter(description = "A JSON value representing the alarm data query. See API call notes above for more details.") @RequestBody AlarmDataQuery query) throws ThingsboardException { @@ -107,13 +98,12 @@ public class EntityQueryController extends BaseController { checkUserId(assigneeId, Operation.READ); } resolveQuery(query); - return this.entityQueryService.findAlarmDataByQuery(getCurrentUser(), query); + return entityQueryService.findAlarmDataByQuery(getCurrentUser(), query); } @ApiOperation(value = "Count Alarms by Query (countAlarmsByQuery)", notes = "Returns the number of alarms that match the query definition.") @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/alarmsQuery/count", method = RequestMethod.POST) - @ResponseBody + @PostMapping("/alarmsQuery/count") public long countAlarmsByQuery(@Parameter(description = "A JSON value representing the alarm count query.") @RequestBody AlarmCountQuery query) throws ThingsboardException { checkNotNull(query); @@ -122,31 +112,47 @@ public class EntityQueryController extends BaseController { checkUserId(assigneeId, Operation.READ); } resolveQuery(query); - return this.entityQueryService.countAlarmsByQuery(getCurrentUser(), query); + return entityQueryService.countAlarmsByQuery(getCurrentUser(), query); } - @ApiOperation(value = "Find Entity Keys by Query", - notes = "Uses entity data query (see 'Find Entity Data by Query') to find first 100 entities. Then fetch and return all unique time-series and/or attribute keys. Used mostly for UI hints.") + @ApiOperation( + value = "Find Available Entity Keys by Query", + notes = """ + Returns unique time series and/or attribute key names from entities matching the query.\n + Executes the Entity Data Query to find up to 100 entities, then fetches and aggregates all distinct key names.\n + Primarily used for UI features like autocomplete suggestions.""" + TENANT_OR_CUSTOMER_AUTHORITY_PARAGRAPH + ) @PreAuthorize("hasAnyAuthority('TENANT_ADMIN', 'CUSTOMER_USER')") - @RequestMapping(value = "/entitiesQuery/find/keys", method = RequestMethod.POST) - @ResponseBody - public DeferredResult findEntityTimeseriesAndAttributesKeysByQuery( - @Parameter(description = "A JSON value representing the entity data query. See API call notes above for more details.") + @PostMapping("/entitiesQuery/find/keys") + public DeferredResult findAvailableEntityKeysByQuery( + @Parameter(description = "Entity data query to find entities. Page size is capped at 100.") @RequestBody EntityDataQuery query, - @Parameter(description = "Include all unique time-series keys to the result.") - @RequestParam("timeseries") boolean isTimeseries, - @Parameter(description = "Include all unique attribute keys to the result.") - @RequestParam("attributes") boolean isAttributes, - @Parameter(description = ATTRIBUTES_SCOPE_DESCRIPTION, schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) - @RequestParam(value = "scope", required = false) String scope) throws ThingsboardException { - TenantId tenantId = getTenantId(); - checkNotNull(query); + + // fixme: combination of timeseries = false and attributes = false is allowed, but always results in empty response, therefore does not make any sense + // such combinations should NOT be allowed, but changing this will break clients + + @Parameter(description = """ + When true, includes unique time series key names in the response. + When false, the 'timeseries' list will be empty.""") + @RequestParam("timeseries") boolean includeTimeseries, + + @Parameter(description = """ + When true, includes unique attribute key names in the response. + When false, the 'attribute' list will be empty. Use 'scope' parameter to filter by attribute scope.""") + @RequestParam("attributes") boolean includeAttributes, + + @Parameter(description = """ + Filters attribute keys by scope. Only applies when 'attributes' is true. + If not specified, returns attribute keys from all scopes.""", + schema = @Schema(allowableValues = {"SERVER_SCOPE", "SHARED_SCOPE", "CLIENT_SCOPE"})) + @RequestParam(value = "scope", required = false) AttributeScope scope + ) throws ThingsboardException { resolveQuery(query); EntityDataPageLink pageLink = query.getPageLink(); if (pageLink.getPageSize() > MAX_PAGE_SIZE) { pageLink.setPageSize(MAX_PAGE_SIZE); } - return entityQueryService.getKeysByQuery(getCurrentUser(), tenantId, query, isTimeseries, isAttributes, scope); + return wrapFuture(entityQueryService.getKeysByQuery(getCurrentUser(), getTenantId(), query, includeTimeseries, includeAttributes, scope)); } @PreAuthorize("hasAnyAuthority('SYS_ADMIN')") diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 9fcb7425b2..1827b986fd 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -69,6 +69,7 @@ import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -82,6 +83,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Consumer; +import java.util.function.Function; import static org.thingsboard.server.service.state.DefaultDeviceStateService.ACTIVITY_STATE; import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAST_CONNECT_TIME; @@ -94,6 +96,7 @@ import static org.thingsboard.server.service.state.DefaultDeviceStateService.LAS public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase implements EdgeRpcService { private final ConcurrentMap sessions = new ConcurrentHashMap<>(); + private final ConcurrentMap sessionsById = new ConcurrentHashMap<>(); private final ConcurrentMap sessionNewEventsLocks = new ConcurrentHashMap<>(); private final Map sessionNewEvents = new HashMap<>(); private final ConcurrentMap> sessionEdgeEventChecks = new ConcurrentHashMap<>(); @@ -283,6 +286,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i destroySession(session); session.cleanUp(); sessions.remove(edgeId); + sessionsById.remove(session.getSessionId()); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); try { @@ -332,9 +336,15 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i TenantId tenantId = edge.getTenantId(); log.info("[{}][{}] edge [{}] connected successfully.", tenantId, edgeGrpcSession.getSessionId(), edgeId); if (sessions.containsKey(edgeId)) { - destroySession(sessions.get(edgeId)); + EdgeGrpcSession existing = sessions.get(edgeId); + if (existing != null) { + log.info("[{}][{}] Replacing existing session [{}] for edge [{}]", tenantId, edgeGrpcSession.getSessionId(), existing.getSessionId(), edgeId); + destroySession(existing); + sessionsById.remove(existing.getSessionId()); + } } sessions.put(edgeId, edgeGrpcSession); + sessionsById.put(edgeGrpcSession.getSessionId(), edgeGrpcSession); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); try { @@ -492,9 +502,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void onEdgeDisconnect(Edge edge, UUID sessionId) { EdgeId edgeId = edge.getId(); log.info("[{}][{}] edge disconnected!", edgeId, sessionId); - EdgeGrpcSession toRemove = sessions.get(edgeId); - if (toRemove.getSessionId().equals(sessionId)) { - toRemove = sessions.remove(edgeId); + EdgeGrpcSession current = sessions.get(edgeId); + if (current != null && current.getSessionId().equals(sessionId)) { + EdgeGrpcSession toRemove = sessions.remove(edgeId); final Lock newEventLock = sessionNewEventsLocks.computeIfAbsent(edgeId, id -> new ReentrantLock()); newEventLock.lock(); try { @@ -503,6 +513,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i newEventLock.unlock(); } destroySession(toRemove); + sessionsById.remove(sessionId); TenantId tenantId = toRemove.getEdge().getTenantId(); save(tenantId, edgeId, ACTIVITY_STATE, false); long lastDisconnectTs = System.currentTimeMillis(); @@ -510,7 +521,18 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i pushRuleEngineMessage(toRemove.getEdge().getTenantId(), edge, lastDisconnectTs, TbMsgType.DISCONNECT_EVENT); cancelScheduleEdgeEventsCheck(edgeId); } else { - log.debug("[{}] edge session [{}] is not available anymore, nothing to remove. most probably this session is already outdated!", edgeId, sessionId); + log.info("[{}] edge session [{}] is not current anymore. Attempting to destroy it by sessionId.", edgeId, sessionId); + EdgeGrpcSession stale = sessionsById.remove(sessionId); + if (stale != null) { + try { + destroySession(stale); + log.info("[{}][{}] Successfully destroyed stale session for edge [{}]", stale.getTenantId(), sessionId, edgeId); + } catch (Exception e) { + log.warn("[{}][{}] Failed to destroy stale session for edge [{}]", stale.getTenantId(), sessionId, edgeId, e); + } + } else { + log.debug("[{}] No session found by sessionId [{}] to destroy", edgeId, sessionId); + } } edgeIdServiceIdCache.evict(edgeId); } @@ -522,6 +544,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i session.getTenantId(), session.getEdge().getId(), session.getEdge().getName(), session.getSessionId()); zombieSessions.add(session); } + } catch (Exception e) { + log.warn("[{}][{}] Exception during session destroy for edge [{}] with session id [{}]", + session.getTenantId(), session.getEdge().getId(), session.getEdge().getName(), session.getSessionId(), e); } } @@ -631,25 +656,9 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private void cleanupZombieSessions() { try { - List toRemove = new ArrayList<>(); - for (EdgeGrpcSession session : sessions.values()) { - if (session instanceof KafkaEdgeGrpcSession kafkaSession && - !kafkaSession.isConnected() && - kafkaSession.getConsumer() != null && - kafkaSession.getConsumer().getConsumer() != null && - !kafkaSession.getConsumer().getConsumer().isStopped()) { - toRemove.add(kafkaSession.getEdge().getId()); - } - } - for (EdgeId edgeId : toRemove) { - log.info("[{}] Destroying session for edge because edge is not connected", edgeId); - EdgeGrpcSession removed = sessions.get(edgeId); - if (removed instanceof KafkaEdgeGrpcSession kafkaSession) { - if (kafkaSession.destroy()) { - sessions.remove(edgeId); - } - } - } + tryToDestroyZombieSessions(getZombieSessions(sessions.values()), s -> sessions.remove(s.getEdge().getId())); + tryToDestroyZombieSessions(getZombieSessions(sessionsById.values()), s -> sessionsById.remove(s.getSessionId())); + zombieSessions.removeIf(zombie -> { if (zombie.destroy()) { log.info("[{}][{}] Successfully cleaned up zombie session [{}] for edge [{}].", @@ -665,4 +674,38 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i log.warn("Failed to cleanup kafka sessions", e); } } + + private List getZombieSessions(Collection sessions) { + List result = new ArrayList<>(); + for (EdgeGrpcSession session : sessions) { + if (isKafkaSessionAndZombie(session)) { + result.add(session); + } + } + return result; + } + + private void tryToDestroyZombieSessions(List sessionsToRemove, Function removeFunc) { + for (EdgeGrpcSession toRemove : sessionsToRemove) { + log.info("[{}] Destroying session for edge because edge is not connected", toRemove.getEdge().getId()); + if (toRemove.destroy()) { + removeFunc.apply(toRemove); + } + } + } + + private boolean isKafkaSessionAndZombie(EdgeGrpcSession session) { + if (session instanceof KafkaEdgeGrpcSession kafkaSession) { + log.debug("[{}] kafkaSession.isConnected() = {}, kafkaSession.getConsumer().getConsumer().isStopped() = {}", + kafkaSession.getEdge().getId(), + kafkaSession.isConnected(), + kafkaSession.getConsumer() != null ? kafkaSession.getConsumer().getConsumer() != null ? kafkaSession.getConsumer().getConsumer().isStopped() : null : null); + return !kafkaSession.isConnected() && + kafkaSession.getConsumer() != null && + kafkaSession.getConsumer().getConsumer() != null && + !kafkaSession.getConsumer().getConsumer().isStopped(); + } + return false; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java index d165be33d4..63669a8e3d 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/KafkaEdgeGrpcSession.java @@ -101,8 +101,21 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession { @Override public ListenableFuture processEdgeEvents() { + if (!isConnected() || isSyncInProgress() || isHighPriorityProcessing) { + log.warn("[{}][{}] Session is not ready (connected={}, syncInProgress={}, highPriority={}), skip starting edge event consumer", + tenantId, edge != null ? edge.getId() : null, isConnected(), isSyncInProgress(), isHighPriorityProcessing); + return Futures.immediateFuture(Boolean.FALSE); + } if (consumer == null || (consumer.getConsumer() != null && consumer.getConsumer().isStopped())) { try { + if (consumerExecutor != null && !consumerExecutor.isShutdown()) { + try { + consumerExecutor.shutdown(); + awaitConsumerTermination(); + } catch (Exception e) { + log.warn("[{}][{}] Failed to shutdown previous consumer executor", tenantId, edge.getId(), e); + } + } this.consumerExecutor = Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName("edge-event-consumer")); this.consumer = QueueConsumerManager.>builder() .name("TB Edge events [" + edge.getId() + "]") @@ -133,6 +146,7 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession { public boolean destroy() { try { if (consumer != null) { + log.info("[{}][{}] Stopping edge event consumer...", tenantId, edge != null ? edge.getId() : null); consumer.stop(); } } catch (Exception e) { @@ -141,16 +155,25 @@ public class KafkaEdgeGrpcSession extends EdgeGrpcSession { } consumer = null; try { - if (consumerExecutor != null) { + if (consumerExecutor != null && !consumerExecutor.isShutdown()) { consumerExecutor.shutdown(); + awaitConsumerTermination(); } } catch (Exception e) { - log.warn("[{}][{}] Failed to shutdown consumer executor", tenantId, edge.getId(), e); + log.warn("[{}][{}] Failed to shutdown edge event consumer executor", tenantId, edge.getId(), e); return false; } return true; } + private void awaitConsumerTermination() { + try { + consumerExecutor.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS); + } catch (InterruptedException ie) { + log.warn("[{}][{}] Interrupted while awaiting consumer executor termination", tenantId, edge.getId()); + } + } + @Override public void cleanUp() { String topic = topicService.buildEdgeEventNotificationsTopicPartitionInfo(tenantId, edge.getId()).getTopic(); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java index 52b33e5297..42057099a8 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/BaseDashboardProcessor.java @@ -65,12 +65,12 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { Dashboard savedDashboard = edgeCtx.getDashboardService().saveDashboard(dashboard, false); - updateDashboardAssignments(tenantId, dashboardById, savedDashboard, newAssignedCustomers); + updateDashboardAssignments(tenantId, customerId, dashboardById, savedDashboard, newAssignedCustomers); return created; } - private void updateDashboardAssignments(TenantId tenantId, Dashboard dashboardById, Dashboard savedDashboard, Set newAssignedCustomers) { + private void updateDashboardAssignments(TenantId tenantId, CustomerId edgeCustomerId, Dashboard dashboardById, Dashboard savedDashboard, Set newAssignedCustomers) { Set currentAssignedCustomers = new HashSet<>(); if (dashboardById != null) { if (dashboardById.getAssignedCustomers() != null) { @@ -78,7 +78,7 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { } } - newAssignedCustomers = filterNonExistingCustomers(tenantId, currentAssignedCustomers, newAssignedCustomers); + newAssignedCustomers = filterNonExistingCustomers(tenantId, edgeCustomerId, currentAssignedCustomers, newAssignedCustomers); Set addedCustomerIds = new HashSet<>(); Set removedCustomerIds = new HashSet<>(); @@ -114,6 +114,6 @@ public abstract class BaseDashboardProcessor extends BaseEdgeProcessor { } } - protected abstract Set filterNonExistingCustomers(TenantId tenantId, Set currentAssignedCustomers, Set newAssignedCustomers); + protected abstract Set filterNonExistingCustomers(TenantId tenantId, CustomerId customerId, Set currentAssignedCustomers, Set newAssignedCustomers); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java index 522c2ba477..b38a9d618e 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/dashboard/DashboardEdgeProcessor.java @@ -25,6 +25,7 @@ import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.msg.TbMsgType; @@ -36,8 +37,10 @@ import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import java.util.HashSet; import java.util.Set; import java.util.UUID; +import java.util.stream.Collectors; @Slf4j @Component @@ -116,14 +119,24 @@ public class DashboardEdgeProcessor extends BaseDashboardProcessor implements Da } @Override - protected Set filterNonExistingCustomers(TenantId tenantId, Set currentAssignedCustomers, Set newAssignedCustomers) { - newAssignedCustomers.addAll(currentAssignedCustomers); - return newAssignedCustomers; + protected Set filterNonExistingCustomers(TenantId tenantId, CustomerId edgeCustomerId, Set currentAssignedCustomers, Set newAssignedCustomers) { + boolean edgeCustomerPresentInNewAssignments = newAssignedCustomers.stream() + .map(ShortCustomerInfo::getCustomerId) + .anyMatch(edgeCustomerId::equals); + + if (edgeCustomerPresentInNewAssignments) { + Set result = new HashSet<>(newAssignedCustomers); + result.addAll(currentAssignedCustomers); + return result; + } else { + return currentAssignedCustomers.stream() + .filter(info -> !edgeCustomerId.equals(info.getCustomerId())) + .collect(Collectors.toSet()); + } } @Override public EdgeEventType getEdgeEventType() { return EdgeEventType.DASHBOARD; } - } diff --git a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java index f39769526a..6ef6b239e5 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/DefaultEntityQueryService.java @@ -15,24 +15,18 @@ */ package org.thingsboard.server.service.query; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.extern.slf4j.Slf4j; -import org.checkerframework.checker.nullness.qual.Nullable; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; -import org.springframework.web.context.request.async.DeferredResult; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.KvUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.AttributeKvEntry; @@ -40,6 +34,7 @@ import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.ComplexFilterPredicate; import org.thingsboard.server.common.data.query.DynamicValue; import org.thingsboard.server.common.data.query.EntityCountQuery; @@ -56,16 +51,13 @@ import org.thingsboard.server.common.data.query.SimpleKeyFilterPredicate; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.entity.EntityService; -import org.thingsboard.server.dao.model.ModelConstants; import org.thingsboard.server.dao.sql.query.EntityKeyMapping; import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.executors.DbCallbackExecutorService; -import org.thingsboard.server.service.security.AccessValidator; import org.thingsboard.server.service.security.model.SecurityUser; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -73,9 +65,10 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutionException; -import java.util.function.Consumer; import java.util.stream.Collectors; +import static com.google.common.util.concurrent.Futures.immediateFuture; + @Service @Slf4j @TbCoreComponent @@ -138,20 +131,12 @@ public class DefaultEntityQueryService implements EntityQueryService { } private void resolveDynamicValue(DynamicValue dynamicValue, SecurityUser user, FilterPredicateType predicateType) { - EntityId entityId; - switch (dynamicValue.getSourceType()) { - case CURRENT_TENANT: - entityId = user.getTenantId(); - break; - case CURRENT_CUSTOMER: - entityId = user.getCustomerId(); - break; - case CURRENT_USER: - entityId = user.getId(); - break; - default: - throw new RuntimeException("Not supported operation for source type: {" + dynamicValue.getSourceType() + "}"); - } + EntityId entityId = switch (dynamicValue.getSourceType()) { + case CURRENT_TENANT -> user.getTenantId(); + case CURRENT_CUSTOMER -> user.getCustomerId(); + case CURRENT_USER -> user.getId(); + default -> throw new RuntimeException("Not supported operation for source type: {" + dynamicValue.getSourceType() + "}"); + }; try { Optional valueOpt = attributesService.find(user.getTenantId(), entityId, @@ -242,101 +227,51 @@ public class DefaultEntityQueryService implements EntityQueryService { } @Override - public DeferredResult getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, - boolean isTimeseries, boolean isAttributes, String attributesScope) { - final DeferredResult response = new DeferredResult<>(); + public ListenableFuture getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, + boolean isTimeseries, boolean isAttributes, AttributeScope scope) { if (!isAttributes && !isTimeseries) { - replyWithEmptyResponse(response); - return response; + return immediateFuture(AvailableEntityKeys.none()); } - List ids = this.findEntityDataByQuery(securityUser, query).getData().stream() + List ids = findEntityDataByQuery(securityUser, query).getData().stream() .map(EntityData::getEntityId) - .collect(Collectors.toList()); + .toList(); if (ids.isEmpty()) { - replyWithEmptyResponse(response); - return response; + return immediateFuture(AvailableEntityKeys.none()); } Set types = ids.stream().map(EntityId::getEntityType).collect(Collectors.toSet()); - final ListenableFuture> timeseriesKeysFuture; - final ListenableFuture> attributesKeysFuture; + ListenableFuture> timeseriesKeysFuture; + ListenableFuture> attributesKeysFuture; if (isTimeseries) { - timeseriesKeysFuture = dbCallbackExecutor.submit(() -> timeseriesService.findAllKeysByEntityIds(tenantId, ids)); + timeseriesKeysFuture = timeseriesService.findAllKeysByEntityIdsAsync(tenantId, ids); } else { - timeseriesKeysFuture = null; + timeseriesKeysFuture = immediateFuture(Collections.emptyList()); } if (isAttributes) { Map> typesMap = ids.stream().collect(Collectors.groupingBy(EntityId::getEntityType)); List>> futures = new ArrayList<>(typesMap.size()); - typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, entityIds, attributesScope)))); + typesMap.forEach((type, entityIds) -> futures.add(dbCallbackExecutor.submit(() -> attributesService.findAllKeysByEntityIds(tenantId, entityIds, scope)))); attributesKeysFuture = Futures.transform(Futures.allAsList(futures), lists -> { if (CollectionUtils.isEmpty(lists)) { return Collections.emptyList(); } - return lists.stream().flatMap(List::stream).distinct().sorted().collect(Collectors.toList()); - }, dbCallbackExecutor); - } else { - attributesKeysFuture = null; - } - - if (isTimeseries && isAttributes) { - Futures.whenAllComplete(timeseriesKeysFuture, attributesKeysFuture).run(() -> { - try { - replyWithResponse(response, types, timeseriesKeysFuture.get(), attributesKeysFuture.get()); - } catch (Exception e) { - log.error("Failed to fetch timeseries and attributes keys!", e); - AccessValidator.handleError(e, response, HttpStatus.INTERNAL_SERVER_ERROR); - } + return lists.stream().flatMap(List::stream).distinct().sorted().toList(); }, dbCallbackExecutor); - } else if (isTimeseries) { - addCallback(timeseriesKeysFuture, keys -> replyWithResponse(response, types, keys, null), - error -> { - log.error("Failed to fetch timeseries keys!", error); - AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR); - }); } else { - addCallback(attributesKeysFuture, keys -> replyWithResponse(response, types, null, keys), - error -> { - log.error("Failed to fetch attributes keys!", error); - AccessValidator.handleError(error, response, HttpStatus.INTERNAL_SERVER_ERROR); - }); + attributesKeysFuture = immediateFuture(Collections.emptyList()); } - return response; - } - - private void replyWithResponse(DeferredResult response, Set types, List timeseriesKeys, List attributesKeys) { - ObjectNode json = JacksonUtil.newObjectNode(); - addItemsToArrayNode(json.putArray("entityTypes"), types); - addItemsToArrayNode(json.putArray("timeseries"), timeseriesKeys); - addItemsToArrayNode(json.putArray("attribute"), attributesKeys); - response.setResult(new ResponseEntity<>(json, HttpStatus.OK)); - } - private void replyWithEmptyResponse(DeferredResult response) { - replyWithResponse(response, Collections.emptySet(), Collections.emptyList(), Collections.emptyList()); - } - - private void addItemsToArrayNode(ArrayNode arrayNode, Collection collection) { - if (!CollectionUtils.isEmpty(collection)) { - collection.forEach(item -> arrayNode.add(item.toString())); - } - } - - private void addCallback(ListenableFuture> future, Consumer> success, Consumer error) { - Futures.addCallback(future, new FutureCallback>() { - @Override - public void onSuccess(@Nullable List keys) { - success.accept(keys); - } - - @Override - public void onFailure(Throwable t) { - error.accept(t); - } - }, dbCallbackExecutor); + return Futures.whenAllComplete(timeseriesKeysFuture, attributesKeysFuture) + .call(() -> { + try { + return new AvailableEntityKeys(types, Futures.getDone(timeseriesKeysFuture), Futures.getDone(attributesKeysFuture)); + } catch (ExecutionException e) { + throw new ThingsboardException(e.getCause(), ThingsboardErrorCode.DATABASE); + } + }, dbCallbackExecutor); } } diff --git a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java index 78ea2519fd..ac6553d738 100644 --- a/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java +++ b/application/src/main/java/org/thingsboard/server/service/query/EntityQueryService.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.service.query; -import org.springframework.http.ResponseEntity; -import org.springframework.web.context.request.async.DeferredResult; +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -37,7 +38,7 @@ public interface EntityQueryService { long countAlarmsByQuery(SecurityUser securityUser, AlarmCountQuery query); - DeferredResult getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, - boolean isTimeseries, boolean isAttributes, String attributesScope); + ListenableFuture getKeysByQuery(SecurityUser securityUser, TenantId tenantId, EntityDataQuery query, + boolean isTimeseries, boolean isAttributes, AttributeScope scope); } diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index fdf54d0109..724b1c622f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -321,9 +321,10 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { "\n" + " # Environment variables\n" + " environment:\n" + - " - host=host.docker.internal\n" + - " - port=1883\n" + - " - accessToken=" + credentials.getCredentialsId() + "\n" + + " - TB_GW_HOST=host.docker.internal\n" + + " - TB_GW_PORT=1883\n" + + " - TB_GW_SECURITY_TYPE=accessToken\n" + + " - TB_GW_ACCESS_TOKEN=" + credentials.getCredentialsId() + "\n" + "\n" + " # Volumes bind\n" + " volumes:\n" + diff --git a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java index 8150456efb..1c2aca46ec 100644 --- a/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/DashboardEdgeTest.java @@ -27,11 +27,15 @@ import org.thingsboard.server.common.data.DashboardInfo; import org.thingsboard.server.common.data.ShortCustomerInfo; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.DashboardId; +import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.edge.v1.CustomerUpdateMsg; import org.thingsboard.server.gen.edge.v1.DashboardUpdateMsg; +import org.thingsboard.server.gen.edge.v1.EdgeConfiguration; import org.thingsboard.server.gen.edge.v1.ResourceUpdateMsg; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; import org.thingsboard.server.gen.edge.v1.UplinkMsg; @@ -183,6 +187,22 @@ public class DashboardEdgeTest extends AbstractEdgeTest { customer.setTitle("Edge Customer"); Customer savedCustomer = doPost("/api/customer", customer, Customer.class); + // assign edge to customer + edgeImitator.expectMessageAmount(2); + doPost("/api/customer/" + savedCustomer.getUuidId() + "/edge/" + edge.getUuidId(), Edge.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + Optional edgeConfigurationOpt = edgeImitator.findMessageByType(EdgeConfiguration.class); + Assert.assertTrue(edgeConfigurationOpt.isPresent()); + EdgeConfiguration edgeConfiguration = edgeConfigurationOpt.get(); + Assert.assertEquals(savedCustomer.getUuidId().getMostSignificantBits(), edgeConfiguration.getCustomerIdMSB()); + Assert.assertEquals(savedCustomer.getUuidId().getLeastSignificantBits(), edgeConfiguration.getCustomerIdLSB()); + Optional customerUpdateOpt = edgeImitator.findMessageByType(CustomerUpdateMsg.class); + Assert.assertTrue(customerUpdateOpt.isPresent()); + CustomerUpdateMsg customerUpdateMsg = customerUpdateOpt.get(); + Customer customerMsg = JacksonUtil.fromString(customerUpdateMsg.getEntity(), Customer.class, true); + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, customerUpdateMsg.getMsgType()); + Assert.assertEquals(savedCustomer, customerMsg); + Dashboard dashboard = buildDashboardForUplinkMsg(savedCustomer); // create dashboard on edge @@ -225,6 +245,23 @@ public class DashboardEdgeTest extends AbstractEdgeTest { foundDashboard = doGet("/api/dashboard/" + dashboard.getUuidId(), Dashboard.class); Assert.assertEquals(DASHBOARD_TITLE + " Updated", foundDashboard.getName()); + + // unassign edge from customer + edgeImitator.expectMessageAmount(2); + doDelete("/api/customer/edge/" + edge.getUuidId(), Edge.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + edgeConfigurationOpt = edgeImitator.findMessageByType(EdgeConfiguration.class); + Assert.assertTrue(edgeConfigurationOpt.isPresent()); + edgeConfiguration = edgeConfigurationOpt.get(); + Assert.assertEquals( + new CustomerId(EntityId.NULL_UUID), + new CustomerId(new UUID(edgeConfiguration.getCustomerIdMSB(), edgeConfiguration.getCustomerIdLSB()))); + customerUpdateOpt = edgeImitator.findMessageByType(CustomerUpdateMsg.class); + Assert.assertTrue(customerUpdateOpt.isPresent()); + customerUpdateMsg = customerUpdateOpt.get(); + Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, customerUpdateMsg.getMsgType()); + Assert.assertEquals(savedCustomer.getUuidId().getMostSignificantBits(), customerUpdateMsg.getIdMSB()); + Assert.assertEquals(savedCustomer.getUuidId().getLeastSignificantBits(), customerUpdateMsg.getIdLSB()); } @Test diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java index 0d5d3dcd13..2cdad499de 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/attributes/AttributesService.java @@ -48,7 +48,7 @@ public interface AttributesService { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); - List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope); + List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope); int removeAllByEntityId(TenantId tenantId, EntityId entityId); diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java index e239e22ee9..0b88ce17cc 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesService.java @@ -63,5 +63,8 @@ public interface TimeseriesService { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds); + void cleanup(long systemTtl); + } diff --git a/common/data/pom.xml b/common/data/pom.xml index 779565ed8e..3afd7cf779 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -113,7 +113,7 @@ compile - dev.langchain4j + org.thingsboard.langchain4j langchain4j-core diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java new file mode 100644 index 0000000000..4cac646908 --- /dev/null +++ b/common/data/src/main/java/org/thingsboard/server/common/data/query/AvailableEntityKeys.java @@ -0,0 +1,67 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.data.query; + +import io.swagger.v3.oas.annotations.media.ArraySchema; +import io.swagger.v3.oas.annotations.media.Schema; +import org.thingsboard.server.common.data.EntityType; + +import java.util.List; +import java.util.Set; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.Objects.requireNonNullElse; + +@Schema( + description = "Contains unique time series and attribute key names discovered from entities matching a query. Used primarily for UI hints such as autocomplete suggestions." +) +public record AvailableEntityKeys( + @Schema( + description = "Set of entity types found among the matched entities.", + example = "[\"DEVICE\", \"ASSET\"]", + requiredMode = Schema.RequiredMode.REQUIRED + ) + Set entityTypes, + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @ArraySchema( + arraySchema = @Schema(description = "List of unique time series key names available on the matched entities."), + schema = @Schema(implementation = String.class, example = "temperature"), + uniqueItems = true + ) + List timeseries, + + @Schema(requiredMode = Schema.RequiredMode.REQUIRED) + @ArraySchema( + arraySchema = @Schema(description = "List of unique attribute key names available on the matched entities."), + schema = @Schema(implementation = String.class, example = "serialNumber"), + uniqueItems = true + ) + List attribute +) { + + public AvailableEntityKeys { + entityTypes = requireNonNullElse(entityTypes, emptySet()); + timeseries = requireNonNullElse(timeseries, emptyList()); + attribute = requireNonNullElse(attribute, emptyList()); + } + + public static AvailableEntityKeys none() { + return new AvailableEntityKeys(emptySet(), emptyList(), emptyList()); + } + +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java index 6542492238..a2ad982765 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/session/AbstractGatewaySessionHandler.java @@ -105,6 +105,7 @@ public abstract class AbstractGatewaySessionHandler { ack(msg, MqttReasonCodes.PubAck.SUCCESS); log.trace("[{}][{}][{}] onDeviceConnectOk: [{}]", gateway.getTenantId(), gateway.getDeviceId(), sessionId, deviceName); }, - t -> logDeviceCreationError(t, deviceName)); + t -> processFailure(msgId, deviceName, CONNECT, ackSent, t)); } public void onDeviceUpdate(TransportProtos.SessionInfoProto sessionInfo, Device device, Optional deviceProfileOpt) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java index 5527d17add..a1af985887 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/AttributesDao.java @@ -53,7 +53,7 @@ public interface AttributesDao { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); - List findAllKeysByEntityIdsAndAttributeType(TenantId tenantId, List entityIds, String attributeType); + List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope); List> removeAllByEntityId(TenantId tenantId, EntityId entityId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java index 62b35caa7f..362aa95ee6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/BaseAttributesService.java @@ -28,7 +28,6 @@ import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.edqs.AttributeKv; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -93,11 +92,11 @@ public class BaseAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope) { - if (StringUtils.isEmpty(scope)) { + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope) { + if (scope == null) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } else { - return attributesDao.findAllKeysByEntityIdsAndAttributeType(tenantId, entityIds, scope); + return attributesDao.findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java index 44b2daaf8e..11ba48773d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/attributes/CachedAttributesService.java @@ -212,11 +212,11 @@ public class CachedAttributesService implements AttributesService { } @Override - public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, String scope) { - if (StringUtils.isEmpty(scope)) { + public List findAllKeysByEntityIds(TenantId tenantId, List entityIds, AttributeScope scope) { + if (scope == null) { return attributesDao.findAllKeysByEntityIds(tenantId, entityIds); } else { - return attributesDao.findAllKeysByEntityIdsAndAttributeType(tenantId, entityIds, scope); + return attributesDao.findAllKeysByEntityIdsAndScope(tenantId, entityIds, scope); } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java index 0a8b8f6399..1e58b23a1f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/attributes/JpaAttributeDao.java @@ -177,10 +177,12 @@ public class JpaAttributeDao extends JpaAbstractDaoListeningExecutorService impl } @Override - public List findAllKeysByEntityIdsAndAttributeType(TenantId tenantId, List entityIds, String attributeType) { + public List findAllKeysByEntityIdsAndScope(TenantId tenantId, List entityIds, AttributeScope scope) { return attributeKvRepository - .findAllKeysByEntityIdsAndAttributeType(entityIds.stream().map(EntityId::getId).collect(Collectors.toList()), AttributeScope.valueOf(attributeType).getId()) - .stream().map(id -> keyDictionaryDao.getKey(id)).collect(Collectors.toList()); + .findAllKeysByEntityIdsAndAttributeType(entityIds.stream().map(EntityId::getId).toList(), scope.getId()) + .stream() + .map(keyDictionaryDao::getKey) + .toList(); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java index d45182442a..bac5529249 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/CachedRedisSqlTimeseriesLatestDao.java @@ -167,5 +167,9 @@ public class CachedRedisSqlTimeseriesLatestDao extends BaseAbstractSqlTimeseries return sqlDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return sqlDao.findAllKeysByEntityIdsAsync(tenantId, entityIds); + } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index c546fc21ea..27470bfe8a 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -24,7 +24,6 @@ import jakarta.annotation.PreDestroy; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.data.domain.Page; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.id.DeviceProfileId; import org.thingsboard.server.common.data.id.EntityId; @@ -38,8 +37,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.dictionary.KeyDictionaryDao; @@ -64,7 +61,6 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.function.Function; -import java.util.stream.Collectors; @Slf4j @Component @@ -185,9 +181,13 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme @Override public List findAllKeysByEntityIds(TenantId tenantId, List entityIds) { - return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).collect(Collectors.toList())); + return tsKvLatestRepository.findAllKeysByEntityIds(entityIds.stream().map(EntityId::getId).toList()); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return service.submit(() -> findAllKeysByEntityIds(tenantId, entityIds)); + } private ListenableFuture getNewLatestEntryFuture(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query, Long version) { ListenableFuture> future = findNewLatestEntryFuture(tenantId, entityId, query); @@ -211,7 +211,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme ReadTsKvQueryResult::getData, MoreExecutors.directExecutor()); } - protected TsKvEntry doFindLatestSync(EntityId entityId, String key) { + protected TsKvEntry doFindLatestSync(EntityId entityId, String key) { TsKvLatestCompositeKey compositeKey = new TsKvLatestCompositeKey( entityId.getId(), diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index ceb7fcf822..de197bf88f 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -156,6 +156,11 @@ public class BaseTimeseriesService implements TimeseriesService { return timeseriesLatestDao.findAllKeysByEntityIds(tenantId, entityIds); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return timeseriesLatestDao.findAllKeysByEntityIdsAsync(tenantId, entityIds); + } + @Override public void cleanup(long systemTtl) { timeseriesDao.cleanup(systemTtl); @@ -300,13 +305,13 @@ public class BaseTimeseriesService implements TimeseriesService { long interval = query.getInterval(); if (interval < 1) { throw new IncorrectParameterException("Invalid TsKvQuery: 'interval' must be greater than 0, but got " + interval + - ". Please check your query parameters and ensure 'endTs' is greater than 'startTs' or increase 'interval'."); + ". Please check your query parameters and ensure 'endTs' is greater than 'startTs' or increase 'interval'."); } long step = Math.max(interval, 1000); long intervalCounts = (query.getEndTs() - query.getStartTs()) / step; if (intervalCounts > maxTsIntervals || intervalCounts < 0) { throw new IncorrectParameterException("Incorrect TsKvQuery. Number of intervals is to high - " + intervalCounts + ". " + - "Please increase 'interval' parameter for your query or reduce the time range of the query."); + "Please increase 'interval' parameter for your query or reduce the time range of the query."); } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java index 54a7e68725..dd44a62349 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesLatestDao.java @@ -36,17 +36,13 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.dao.model.ModelConstants; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import org.thingsboard.server.dao.nosql.TbResultSet; import org.thingsboard.server.dao.sqlts.AggregationTimeseriesDao; import org.thingsboard.server.dao.util.NoSqlTsLatestDao; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import static com.datastax.oss.driver.api.querybuilder.QueryBuilder.literal; @@ -103,6 +99,10 @@ public class CassandraBaseTimeseriesLatestDao extends AbstractCassandraBaseTimes return Collections.emptyList(); } + @Override + public ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds) { + return Futures.immediateFuture(Collections.emptyList()); + } @Override public ListenableFuture saveLatest(TenantId tenantId, EntityId entityId, TsKvEntry tsKvEntry) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java index 32479301ae..74c041e4d3 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesLatestDao.java @@ -22,12 +22,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.kv.DeleteTsKvQuery; import org.thingsboard.server.common.data.kv.TsKvEntry; import org.thingsboard.server.common.data.kv.TsKvLatestRemovingResult; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; -import org.thingsboard.server.dao.model.sqlts.latest.TsKvLatestEntity; import java.util.List; -import java.util.Map; import java.util.Optional; public interface TimeseriesLatestDao { @@ -54,4 +50,6 @@ public interface TimeseriesLatestDao { List findAllKeysByEntityIds(TenantId tenantId, List entityIds); + ListenableFuture> findAllKeysByEntityIdsAsync(TenantId tenantId, List entityIds); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java index cae2d56ae5..46a7b650f7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/util/DeviceConnectivityUtil.java @@ -117,24 +117,26 @@ public class DeviceConnectivityUtil { dockerComposeBuilder.append("\n"); dockerComposeBuilder.append(" # Environment variables\n"); dockerComposeBuilder.append(" environment:\n"); - dockerComposeBuilder.append(" - host=").append(isLocalhost(host) ? HOST_DOCKER_INTERNAL : host).append("\n"); - dockerComposeBuilder.append(" - port=1883\n"); + dockerComposeBuilder.append(" - TB_GW_HOST=").append(isLocalhost(host) ? HOST_DOCKER_INTERNAL : host).append("\n"); + dockerComposeBuilder.append(" - TB_GW_PORT=1883\n"); switch (deviceCredentials.getCredentialsType()) { case ACCESS_TOKEN: - dockerComposeBuilder.append(" - accessToken=").append(deviceCredentials.getCredentialsId()).append("\n"); + dockerComposeBuilder.append(" - TB_GW_SECURITY_TYPE=accessToken\n"); + dockerComposeBuilder.append(" - TB_GW_ACCESS_TOKEN=").append(deviceCredentials.getCredentialsId()).append("\n"); break; case MQTT_BASIC: + dockerComposeBuilder.append(" - TB_GW_SECURITY_TYPE=usernamePassword\n"); BasicMqttCredentials credentials = JacksonUtil.fromString(deviceCredentials.getCredentialsValue(), BasicMqttCredentials.class); if (credentials != null) { if (StringUtils.isNotEmpty(credentials.getClientId())) { - dockerComposeBuilder.append(" - clientId=").append(credentials.getClientId()).append("\n"); + dockerComposeBuilder.append(" - TB_GW_CLIENT_ID=").append(credentials.getClientId()).append("\n"); } if (StringUtils.isNotEmpty(credentials.getUserName())) { - dockerComposeBuilder.append(" - username=").append(credentials.getUserName()).append("\n"); + dockerComposeBuilder.append(" - TB_GW_USERNAME=").append(credentials.getUserName()).append("\n"); } if (StringUtils.isNotEmpty(credentials.getPassword())) { - dockerComposeBuilder.append(" - password=").append(credentials.getPassword()).append("\n"); + dockerComposeBuilder.append(" - TB_GW_PASSWORD=").append(credentials.getPassword()).append("\n"); } } break; diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java index 5978d903c7..41a013e4c9 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/attributes/BaseAttributesServiceTest.java @@ -223,7 +223,7 @@ public abstract class BaseAttributesServiceTest extends AbstractServiceTest { saveAttribute(tenantId, deviceId, AttributeScope.SERVER_SCOPE, "key2", "123"); Awaitility.await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> { - List keys = attributesService.findAllKeysByEntityIds(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE.name()); + List keys = attributesService.findAllKeysByEntityIds(tenantId, List.of(deviceId), AttributeScope.SERVER_SCOPE); assertThat(keys).containsOnly("key1", "key2"); }); } diff --git a/pom.xml b/pom.xml index b0768966bd..4676ee3793 100755 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ 4.0.2 1.7.5 3.8.0 - 1.1.0 + 1.8.0-TB 2.38.0 1.24 1.11.0 @@ -911,7 +911,7 @@ import - dev.langchain4j + org.thingsboard.langchain4j langchain4j-bom ${langchain4j.version} pom @@ -1908,6 +1908,10 @@ central https://repo1.maven.org/maven2/ + + thingsboard-repo + https://repo.thingsboard.io/artifactory/libs-release-public + spring-snapshots Spring Snapshots diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 2c8e01d246..6eb61b3e13 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -39,10 +39,12 @@ import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.rest.client.utils.RestJsonConverter; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.ClaimRequest; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.Dashboard; @@ -160,6 +162,7 @@ import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.query.AlarmCountQuery; import org.thingsboard.server.common.data.query.AlarmData; import org.thingsboard.server.common.data.query.AlarmDataQuery; +import org.thingsboard.server.common.data.query.AvailableEntityKeys; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityData; import org.thingsboard.server.common.data.query.EntityDataQuery; @@ -592,7 +595,7 @@ public class RestClient implements Closeable { } public PageData getAllAlarmsV2(List statusList, List severityList, - List typeList, String assignedId, TimePageLink pageLink) { + List typeList, String assignedId, TimePageLink pageLink) { String urlSecondPart = "/api/v2/alarms?"; Map params = new HashMap<>(); if (!CollectionUtils.isEmpty(statusList)) { @@ -1824,12 +1827,15 @@ public class RestClient implements Closeable { }).getBody(); } - public JsonNode findEntityTimeseriesAndAttributesKeysByQuery(EntityDataQuery query) { - return restTemplate.exchange( - baseURL + "/api/entitiesQuery/find/keys", - HttpMethod.POST, new HttpEntity<>(query), - new ParameterizedTypeReference() { - }).getBody(); + public AvailableEntityKeys findAvailableEntityKeysByQuery(EntityDataQuery query, boolean includeTimeseries, boolean includeAttributes, AttributeScope scope) { + var uri = UriComponentsBuilder.fromUriString(baseURL) + .path("/api/entitiesQuery/find/keys") + .queryParam("timeseries", includeTimeseries) + .queryParam("attributes", includeAttributes) + .queryParamIfPresent("scope", Optional.ofNullable(scope)) + .build() + .toUri(); + return restTemplate.exchange(uri, HttpMethod.POST, new HttpEntity<>(query), new ParameterizedTypeReference() {}).getBody(); } public PageData findAlarmDataByQuery(AlarmDataQuery query) { diff --git a/rule-engine/rule-engine-api/pom.xml b/rule-engine/rule-engine-api/pom.xml index 59397d4a92..0c1bb2e362 100644 --- a/rule-engine/rule-engine-api/pom.xml +++ b/rule-engine/rule-engine-api/pom.xml @@ -99,7 +99,7 @@ provided - dev.langchain4j + org.thingsboard.langchain4j langchain4j diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index 98c7fde665..598377ec86 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -154,7 +154,7 @@ json-path - dev.langchain4j + org.thingsboard.langchain4j langchain4j diff --git a/ui-ngx/package.json b/ui-ngx/package.json index 580b63a588..3403e7a3aa 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -141,6 +141,9 @@ "rollup": "4.22.4", "@babel/core": "7.25.2", "esbuild": "0.23.0", - "jquery.terminal/coveralls-next/form-data": "4.0.4" + "jquery.terminal/coveralls-next/form-data": "4.0.4", + "node-forge": "1.3.3", + "glob": "10.5.0", + "path-to-regexp": "0.1.12" } } diff --git a/ui-ngx/src/app/modules/home/components/ai-model/check-connectivity-dialog.component.ts b/ui-ngx/src/app/modules/home/components/ai-model/check-connectivity-dialog.component.ts index dd2c27cb47..cd593b4291 100644 --- a/ui-ngx/src/app/modules/home/components/ai-model/check-connectivity-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/ai-model/check-connectivity-dialog.component.ts @@ -56,9 +56,7 @@ export class CheckConnectivityDialogComponent extends DialogComponent implements OnInit { +export class AlarmDetailsDialogComponent extends DialogComponent { alarmId: string; alarmFormGroup: UntypedFormGroup; @@ -128,13 +128,12 @@ export class AlarmDetailsDialogComponent extends DialogComponent diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config-dialog.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config-dialog.component.ts index 829402a53f..e455a4aeb9 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config-dialog.component.ts @@ -56,6 +56,7 @@ export interface DataKeyConfigDialogData { hideDataKeyColor?: boolean; hideDataKeyUnits?: boolean; hideDataKeyDecimals?: boolean; + hideDataKeyAggregation?: boolean; supportsUnitConversion?: boolean } diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.html b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.html index 73c1865cc2..1d9e6dcf3d 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.html +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.html @@ -79,7 +79,7 @@ formControlName="funcBody"> - +
{{ 'datakey.aggregation' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.ts b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.ts index c7b65bfc83..8d2e7b7fc5 100644 --- a/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.ts +++ b/ui-ngx/src/app/modules/home/components/widget/lib/settings/common/key/data-key-config.component.ts @@ -153,6 +153,10 @@ export class DataKeyConfigComponent extends PageComponent implements OnInit, Con @coerceBoolean() hideDataKeyDecimals = false; + @Input() + @coerceBoolean() + hideDataKeyAggregation = false; + @Input() @coerceBoolean() supportsUnitConversion = false; diff --git a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.ts b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.ts index 141c274422..efae0d0b62 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.ts +++ b/ui-ngx/src/app/modules/home/pages/mobile/applications/mobile-app.component.ts @@ -176,7 +176,7 @@ export class MobileAppComponent extends EntityComponent { context: ctx, showCloseButton: false, popoverContentStyle: {padding: '16px 24px'}, - isModal: false + isModal: true }); releaseNotesPanelPopover.tbComponentRef.instance.popover = releaseNotesPanelPopover; releaseNotesPanelPopover.tbComponentRef.instance.editorContentApplied.subscribe((releaseNotes) => { diff --git a/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts b/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts index 0b5c93ff59..c177e8282b 100644 --- a/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts +++ b/ui-ngx/src/app/modules/home/pages/mobile/common/editor-panel.component.ts @@ -59,9 +59,33 @@ export class EditorPanelComponent implements OnInit { resize: false, setup: (editor) => { editor.on('PostRender', function() { - const container = editor.getContainer().closest('.tb-popover-content'); - const uiContainer = document.querySelector('.tox.tox-tinymce-aux'); - container.parentNode.appendChild(uiContainer); + const container = document.querySelector('.tox.tox-tinymce-aux'); + const styleSheet = document.createElement('style'); + styleSheet.innerText = ` + .tox-tiered-menu .tox-menu { + width: fit-content; + max-width: min(80%, 440px); + @media screen and (max-width: 510px) { + max-width: calc(100% - 64px); + } + media screen and (min-width: 511px) and (max-width: 548px) { + max-width: calc(100% - 84px); + } + media screen and (min-width: 549px) and (max-width: 599px) { + max-width: calc(100% - 104px); + } + } + .tox-tiered-menu .tox-menu .tox-collection__item-label { + word-break: normal; + } + @media screen and (max-width: 890px) { + .tox-tiered-menu > .tox-collection--list:not(:first-child) { + left: auto !important; + right: 0 !important; + } + } + `; + container.prepend(styleSheet); }); }, relative_urls: false, diff --git a/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-notification-dialog.component.html b/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-notification-dialog.component.html index 95715bef87..560db89dbc 100644 --- a/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-notification-dialog.component.html +++ b/ui-ngx/src/app/modules/home/pages/notification/recipient/recipient-notification-dialog.component.html @@ -70,6 +70,7 @@ { + editor.on('PostRender', function() { + const container = document.querySelector('.tox.tox-tinymce-aux'); + const styleSheet = document.createElement('style'); + styleSheet.innerText = ` + .tox-tiered-menu .tox-menu { + width: fit-content; + max-width: min(80%, 440px); + @media screen and (max-width: 510px) { + max-width: calc(100% - 64px); + } + media screen and (min-width: 511px) and (max-width: 548px) { + max-width: calc(100% - 84px); + } + media screen and (min-width: 549px) and (max-width: 599px) { + max-width: calc(100% - 104px); + } + } + .tox-tiered-menu .tox-menu .tox-collection__item-label { + word-break: normal; + } + @media screen and (max-width: 890px) { + .tox-tiered-menu > .tox-collection--list:not(:first-child) { + left: auto !important; + right: 0 !important; + } + } + `; + container.prepend(styleSheet); + }); + }, relative_urls: false, urlconverter_callback: (url) => url }; diff --git a/ui-ngx/src/app/shared/models/ai-model.models.ts b/ui-ngx/src/app/shared/models/ai-model.models.ts index d3b70f9a74..410f7ad900 100644 --- a/ui-ngx/src/app/shared/models/ai-model.models.ts +++ b/ui-ngx/src/app/shared/models/ai-model.models.ts @@ -110,6 +110,7 @@ export const AiModelMap = new Map