diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 66e5cf6f90..0e21444431 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -228,10 +228,9 @@ public class AuthController extends BaseController { @ApiOperation(value = "Reset password (resetPassword)", notes = "Checks the password reset token and updates the password. " + - "If token is valid, returns the object that contains [JWT](https://jwt.io/) access and refresh tokens. " + "If token is not valid, returns '400 Bad Request'.") @PostMapping(value = "/noauth/resetPassword") - public JwtPair resetPassword(@Parameter(description = "Reset password request.") + public void resetPassword(@Parameter(description = "Reset password request.") @RequestBody ResetPasswordRequest resetPasswordRequest, HttpServletRequest request) throws ThingsboardException { String resetToken = resetPasswordRequest.getResetToken(); @@ -263,8 +262,6 @@ public class AuthController extends BaseController { } eventPublisher.publishEvent(new UserCredentialsInvalidationEvent(securityUser.getId())); - - return tokenFactory.createTokenPair(securityUser); } else { throw new ThingsboardException("Invalid reset token!", ThingsboardErrorCode.BAD_REQUEST_PARAMS); } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java index 9bcd8fc373..d996a1dbc9 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java @@ -507,7 +507,7 @@ public class EdgeMsgConstructorUtils { JsonObject data = entityData.getAsJsonObject(); builder.setPostTelemetryMsg(JsonConverter.convertToTelemetryProto(data.getAsJsonObject("data"), ts)); } catch (Exception e) { - log.warn("[{}][{}] Can't convert to telemetry proto, entityData [{}]", tenantId, entityId, entityData, e); + log.trace("[{}][{}] Can't convert to telemetry proto, entityData [{}]", tenantId, entityId, entityData, e); } break; case ATTRIBUTES_UPDATED: @@ -522,7 +522,7 @@ public class EdgeMsgConstructorUtils { builder.setPostAttributeScope(getScopeOfDefault(data)); builder.setAttributeTs(ts); } catch (Exception e) { - log.warn("[{}][{}] Can't convert to AttributesUpdatedMsg proto, entityData [{}]", tenantId, entityId, entityData, e); + log.trace("[{}][{}] Can't convert to AttributesUpdatedMsg proto, entityData [{}]", tenantId, entityId, entityData, e); } break; case POST_ATTRIBUTES: @@ -533,7 +533,7 @@ public class EdgeMsgConstructorUtils { builder.setPostAttributeScope(getScopeOfDefault(data)); builder.setAttributeTs(ts); } catch (Exception e) { - log.warn("[{}][{}] Can't convert to PostAttributesMsg, entityData [{}]", tenantId, entityId, entityData, e); + log.trace("[{}][{}] Can't convert to PostAttributesMsg, entityData [{}]", tenantId, entityId, entityData, e); } break; case ATTRIBUTES_DELETED: @@ -546,7 +546,7 @@ public class EdgeMsgConstructorUtils { attributeDeleteMsg.build(); builder.setAttributeDeleteMsg(attributeDeleteMsg); } catch (Exception e) { - log.warn("[{}][{}] Can't convert to AttributeDeleteMsg proto, entityData [{}]", tenantId, entityId, entityData, e); + log.trace("[{}][{}] Can't convert to AttributeDeleteMsg proto, entityData [{}]", tenantId, entityId, entityData, e); } break; } diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index ecd0c6d9f3..9ce5973639 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -203,7 +203,7 @@ public abstract class EdgeGrpcSession implements Closeable { @Override public void onError(Throwable t) { - log.error("[{}][{}] Stream was terminated due to error:", tenantId, sessionId, t); + log.trace("[{}][{}] Stream was terminated due to error:", tenantId, sessionId, t); closeSession(); } @@ -255,7 +255,7 @@ public abstract class EdgeGrpcSession implements Closeable { private void doSync(EdgeSyncCursor cursor) { if (cursor.hasNext()) { EdgeEventFetcher next = cursor.getNext(); - log.info("[{}][{}] starting sync process, cursor current idx = {}, class = {}", + log.debug("[{}][{}] starting sync process, cursor current idx = {}, class = {}", tenantId, edge.getId(), cursor.getCurrentIdx(), next.getClass().getSimpleName()); ListenableFuture> future = startProcessingEdgeEvents(next); Futures.addCallback(future, new FutureCallback<>() { @@ -651,7 +651,7 @@ public abstract class EdgeGrpcSession implements Closeable { default -> log.warn("[{}][{}] Unsupported action type [{}]", tenantId, sessionId, edgeEvent.getAction()); } } catch (Exception e) { - log.error("[{}][{}] Exception during converting edge event to downlink msg", tenantId, sessionId, e); + log.trace("[{}][{}] Exception during converting edge event to downlink msg", tenantId, sessionId, e); } if (downlinkMsg != null) { result.add(downlinkMsg); @@ -763,7 +763,7 @@ public abstract class EdgeGrpcSession implements Closeable { try { outputStream.onNext(responseMsg); } catch (Exception e) { - log.error("[{}][{}] Failed to send downlink message [{}]", tenantId, sessionId, downlinkMsgStr, e); + log.trace("[{}][{}] Failed to send downlink message [{}]", tenantId, sessionId, downlinkMsgStr, e); connected = false; sessionCloseListener.accept(edge, sessionId); } finally { @@ -909,7 +909,7 @@ public abstract class EdgeGrpcSession implements Closeable { } } catch (Exception e) { String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); - log.error("[{}][{}] Can't process uplink msg [{}]", edge.getTenantId(), sessionId, uplinkMsg, e); + log.trace("[{}][{}] Can't process uplink msg [{}]", edge.getTenantId(), sessionId, uplinkMsg, e); ctx.getRuleProcessor().process(EdgeCommunicationFailureTrigger.builder().tenantId(edge.getTenantId()).edgeId(edge.getId()) .customerId(edge.getCustomerId()).edgeName(edge.getName()).failureMsg(failureMsg).error(e.getMessage()).build()); return Futures.immediateFailedFuture(e); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java index dd9f9a2b42..351c9b411b 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeSyncCursor.java @@ -65,8 +65,10 @@ public class EdgeSyncCursor { fetchers.add(new AdminSettingsEdgeEventFetcher(ctx.getAdminSettingsService())); fetchers.add(new TenantAdminUsersEdgeEventFetcher(ctx.getUserService())); } - Customer publicCustomer = ctx.getCustomerService().findOrCreatePublicCustomer(edge.getTenantId()); - fetchers.add(new CustomerEdgeEventFetcher(publicCustomer.getId())); + Customer publicCustomer = ctx.getCustomerService().findPublicCustomer(edge.getTenantId()); + if (publicCustomer != null) { + fetchers.add(new CustomerEdgeEventFetcher(publicCustomer.getId())); + } if (edge.getCustomerId() != null && !EntityId.NULL_UUID.equals(edge.getCustomerId().getId())) { fetchers.add(new CustomerEdgeEventFetcher(edge.getCustomerId())); fetchers.add(new CustomerUsersEdgeEventFetcher(ctx.getUserService(), edge.getCustomerId())); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java index 784d10b26c..5f02fbc554 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/customer/CustomerEdgeProcessor.java @@ -81,13 +81,16 @@ public class CustomerEdgeProcessor extends BaseEdgeProcessor { UUID uuid = new UUID(edgeNotificationMsg.getEntityIdMSB(), edgeNotificationMsg.getEntityIdLSB()); CustomerId customerId = new CustomerId(EntityIdFactory.getByEdgeEventTypeAndUuid(type, uuid).getId()); switch (actionType) { - case UPDATED: - List> futures = new ArrayList<>(); - PageDataIterable edges = new PageDataIterable<>(link -> edgeCtx.getEdgeService().findEdgesByTenantIdAndCustomerId(tenantId, customerId, link), 1024); - for (Edge edge : edges) { - futures.add(saveEdgeEvent(tenantId, edge.getId(), type, actionType, customerId, null)); + case ADDED: + Customer customerById = edgeCtx.getCustomerService().findCustomerById(tenantId, customerId); + if (customerById != null && customerById.isPublic()) { + return findEdgesAndSaveEdgeEvents(link -> edgeCtx.getEdgeService().findEdgesByTenantId(tenantId, link), + tenantId, type, actionType, customerId); } - return Futures.transform(Futures.allAsList(futures), voids -> null, dbCallbackExecutorService); + return Futures.immediateFuture(null); + case UPDATED: + return findEdgesAndSaveEdgeEvents(link -> edgeCtx.getEdgeService().findEdgesByTenantIdAndCustomerId(tenantId, customerId, link), + tenantId, type, actionType, customerId); case DELETED: EdgeId edgeId = new EdgeId(new UUID(edgeNotificationMsg.getEdgeIdMSB(), edgeNotificationMsg.getEdgeIdLSB())); return saveEdgeEvent(tenantId, edgeId, type, actionType, customerId, null); @@ -96,6 +99,16 @@ public class CustomerEdgeProcessor extends BaseEdgeProcessor { } } + public ListenableFuture findEdgesAndSaveEdgeEvents(PageDataIterable.FetchFunction edgeFetcher, TenantId tenantId, + EdgeEventType type, EdgeEventActionType actionType, CustomerId customerId) { + List> futures = new ArrayList<>(); + PageDataIterable edges = new PageDataIterable<>(edgeFetcher, 1024); + for (Edge edge : edges) { + futures.add(saveEdgeEvent(tenantId, edge.getId(), type, actionType, customerId, null)); + } + return Futures.transform(Futures.allAsList(futures), voids -> null, dbCallbackExecutorService); + } + @Override public EdgeEventType getEdgeEventType() { return EdgeEventType.CUSTOMER; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java index 92c48a5fed..561dc7122a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbClusterService.java @@ -551,11 +551,15 @@ public class DefaultTbClusterService implements TbClusterService { } private void processEdgeNotification(EdgeId edgeId, ToEdgeNotificationMsg toEdgeNotificationMsg) { - var serviceIdOpt = Optional.ofNullable(edgeIdServiceIdCache.get(edgeId)); - serviceIdOpt.ifPresentOrElse( - serviceId -> pushMsgToEdgeNotification(toEdgeNotificationMsg, serviceId.get()), - () -> broadcastEdgeNotification(edgeId, toEdgeNotificationMsg) - ); + if (edgesEnabled) { + var serviceIdOpt = Optional.ofNullable(edgeIdServiceIdCache.get(edgeId)); + serviceIdOpt.ifPresentOrElse( + serviceId -> pushMsgToEdgeNotification(toEdgeNotificationMsg, serviceId.get()), + () -> broadcastEdgeNotification(edgeId, toEdgeNotificationMsg) + ); + } else { + log.trace("Edges disabled. Ignoring edge notification {} for edgeId: {}", toEdgeNotificationMsg, edgeId); + } } private void pushMsgToEdgeNotification(ToEdgeNotificationMsg toEdgeNotificationMsg, String serviceId) { diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java index 7415b0bdad..d9fffb6137 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbEdgeConsumerService.java @@ -50,6 +50,7 @@ import org.thingsboard.server.queue.discovery.event.PartitionChangeEvent; import org.thingsboard.server.queue.provider.TbCoreQueueFactory; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeContextComponent; +import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.queue.processing.AbstractConsumerService; import org.thingsboard.server.service.queue.processing.IdMsgPair; @@ -194,36 +195,42 @@ public class DefaultTbEdgeConsumerService extends AbstractConsumerService msg, TbCallback callback) { ToEdgeNotificationMsg toEdgeNotificationMsg = msg.getValue(); try { + EdgeRpcService edgeRpcService = edgeCtx.getEdgeRpcService(); + if (edgeRpcService == null) { + log.debug("No EdgeRpcService available (edge functionality disabled), ignoring msg: {}", toEdgeNotificationMsg); + callback.onSuccess(); + return; + } if (toEdgeNotificationMsg.hasEdgeHighPriority()) { EdgeSessionMsg edgeSessionMsg = ProtoUtils.fromProto(toEdgeNotificationMsg.getEdgeHighPriority()); - edgeCtx.getEdgeRpcService().onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); + edgeRpcService.onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); callback.onSuccess(); } else if (toEdgeNotificationMsg.hasEdgeEventUpdate()) { EdgeSessionMsg edgeSessionMsg = ProtoUtils.fromProto(toEdgeNotificationMsg.getEdgeEventUpdate()); - edgeCtx.getEdgeRpcService().onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); + edgeRpcService.onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); callback.onSuccess(); } else if (toEdgeNotificationMsg.hasToEdgeSyncRequest()) { EdgeSessionMsg edgeSessionMsg = ProtoUtils.fromProto(toEdgeNotificationMsg.getToEdgeSyncRequest()); - edgeCtx.getEdgeRpcService().onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); + edgeRpcService.onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); callback.onSuccess(); } else if (toEdgeNotificationMsg.hasFromEdgeSyncResponse()) { EdgeSessionMsg edgeSessionMsg = ProtoUtils.fromProto(toEdgeNotificationMsg.getFromEdgeSyncResponse()); - edgeCtx.getEdgeRpcService().onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); + edgeRpcService.onToEdgeSessionMsg(edgeSessionMsg.getTenantId(), edgeSessionMsg); callback.onSuccess(); } else if (toEdgeNotificationMsg.hasComponentLifecycle()) { ComponentLifecycleMsg componentLifecycle = ProtoUtils.fromProto(toEdgeNotificationMsg.getComponentLifecycle()); TenantId tenantId = componentLifecycle.getTenantId(); EdgeId edgeId = new EdgeId(componentLifecycle.getEntityId().getId()); if (ComponentLifecycleEvent.DELETED.equals(componentLifecycle.getEvent())) { - edgeCtx.getEdgeRpcService().deleteEdge(tenantId, edgeId); + edgeRpcService.deleteEdge(tenantId, edgeId); } else if (ComponentLifecycleEvent.UPDATED.equals(componentLifecycle.getEvent())) { Edge edge = edgeCtx.getEdgeService().findEdgeById(tenantId, edgeId); - edgeCtx.getEdgeRpcService().updateEdge(tenantId, edge); + edgeRpcService.updateEdge(tenantId, edge); } callback.onSuccess(); } } catch (Exception e) { - log.error("Error processing edge notification message", e); + log.error("Error processing edge notification message {}", toEdgeNotificationMsg, e); callback.onFailure(e); } diff --git a/application/src/main/resources/banner.txt b/application/src/main/resources/banner.txt index 57f907c5e0..cf6ed4da7d 100644 --- a/application/src/main/resources/banner.txt +++ b/application/src/main/resources/banner.txt @@ -1,10 +1,10 @@ - ______ __ _ ____ __ - /_ __/ / /_ (_) ____ ____ _ _____ / __ ) ____ ____ _ _____ ____/ / - / / / __ \ / / / __ \ / __ `/ / ___/ / __ | / __ \ / __ `/ / ___/ / __ / - / / / / / / / / / / / / / /_/ / (__ ) / /_/ / / /_/ // /_/ / / / / /_/ / -/_/ /_/ /_/ /_/ /_/ /_/ \__, / /____/ /_____/ \____/ \__,_/ /_/ \__,_/ - /____/ + _____ _ _ ____ _ + |_ _| |__ (_)_ __ __ _ ___| __ ) ___ __ _ _ __ __| | + | | | '_ \| | '_ \ / _` / __| _ \ / _ \ / _` | '__/ _` | + | | | | | | | | | | (_| \__ \ |_) | (_) | (_| | | | (_| | + |_| |_| |_|_|_| |_|\__, |___/____/ \___/ \__,_|_| \__,_| + |___/ - =================================================== + =========================================================== :: ${application.title} :: ${application.formatted-version} - =================================================== + =========================================================== diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 37f2bcd5a7..716797c2d2 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1423,7 +1423,7 @@ device: pem_cert_file: "${DEVICE_CONNECTIVITY_COAPS_CA_ROOT_CERT:cafile.pem}" gateway: # The docker tag for thingsboard/tb-gateway image used in docker-compose file for gateway launch - image_version: "${DEVICE_CONNECTIVITY_GATEWAY_IMAGE_VERSION:latest}" + image_version: "${DEVICE_CONNECTIVITY_GATEWAY_IMAGE_VERSION:3.7-stable}" # Edges parameters edges: @@ -1590,11 +1590,24 @@ queue: # tb_rule_engine.sq: # - key: max.poll.records # value: "${TB_QUEUE_KAFKA_SQ_MAX_POLL_RECORDS:1024}" + tb_edge: + # Properties for consumers targeting edge service update topics. + - key: max.poll.records + # Define the maximum number of records that can be polled from tb_edge topics per request. + value: "${TB_QUEUE_KAFKA_EDGE_EVENTS_MAX_POLL_RECORDS:10}" + tb_edge.notifications: + # Properties for consumers targeting high-priority edge notifications. + # These notifications include RPC calls, lifecycle events, and new queue messages, + # requiring minimal latency and swift processing. + - key: max.poll.records + # Define the maximum number of records that can be polled from tb_edge.notifications. topics. + value: "${TB_QUEUE_KAFKA_EDGE_HP_EVENTS_MAX_POLL_RECORDS:10}" tb_edge_event.notifications: - # Example of specific consumer properties value per topic for edge event + # Properties for consumers targeting downlinks meant for specific edge topics. + # Topic names are dynamically constructed using tenant and edge identifiers. - key: max.poll.records - # Example of specific consumer properties value per topic for edge event - value: "${TB_QUEUE_KAFKA_EDGE_EVENT_MAX_POLL_RECORDS:50}" + # Define the maximum number of records that can be polled from tb_edge_event.notifications.. topics. + value: "${TB_QUEUE_KAFKA_EDGE_NOTIFICATIONS_MAX_POLL_RECORDS:10}" tb_housekeeper: # Consumer properties for Housekeeper tasks topic - key: max.poll.records @@ -1765,9 +1778,13 @@ queue: max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + # Strings longer than this threshold will be compressed + string_compression_length_threshold: "${TB_EDQS_STRING_COMPRESSION_LENGTH_THRESHOLD:512}" stats: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" + # Threshold for slow queries to log, in milliseconds + slow_query_threshold: "${TB_EDQS_SLOW_QUERY_THRESHOLD_MS:3000}" vc: # Default topic name topic: "${TB_QUEUE_VC_TOPIC:tb_version_control}" @@ -1837,11 +1854,13 @@ queue: # Interval in milliseconds to poll messages poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" edge: - # Default topic name + # Topic name to notify edge service on entity updates, assignment, etc. topic: "${TB_QUEUE_EDGE_TOPIC:tb_edge}" - # For high-priority notifications that require minimum latency and processing time + # Topic prefix for high-priority edge notifications (rpc, lifecycle, new messages in queue) that require minimum latency and processing time. + # Each tb-core has its own topic: . notifications_topic: "${TB_QUEUE_EDGE_NOTIFICATIONS_TOPIC:tb_edge.notifications}" - # For edge events messages + # Topic prefix for downlinks to be pushed to specific edge. + # Every edge has its own unique topic: .. event_notifications_topic: "${TB_QUEUE_EDGE_EVENT_NOTIFICATIONS_TOPIC:tb_edge_event.notifications}" # Amount of partitions used by Edge services partitions: "${TB_QUEUE_EDGE_PARTITIONS:10}" diff --git a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java index cb475d7f16..4ccf40415d 100644 --- a/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/DeviceConnectivityControllerTest.java @@ -23,6 +23,7 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.springframework.beans.factory.annotation.Value; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; @@ -94,6 +95,9 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { private DeviceProfileId mqttDeviceProfileId; private DeviceProfileId coapDeviceProfileId; + @Value("${device.connectivity.gateway.image_version:3.7-stable}") + private String gatewayImageVersion; + @Before public void beforeTest() throws Exception { loginSysAdmin(); @@ -298,7 +302,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { "services:\n" + " # ThingsBoard IoT Gateway Service Configuration\n" + " tb-gateway:\n" + - " image: thingsboard/tb-gateway:latest\n" + + " image: thingsboard/tb-gateway:" + gatewayImageVersion + "\n" + " container_name: tb-gateway\n" + " restart: always\n" + "\n" + @@ -847,7 +851,7 @@ public class DeviceConnectivityControllerTest extends AbstractControllerTest { "-t \"application/json\" -e \"{temperature:25}\" coap://test.domain:5683/api/v1/%s/telemetry", credentials.getCredentialsId())); assertThat(linuxCoapCommands.get(COAPS).get(0).asText()).isEqualTo("curl -f -S -o " + CA_ROOT_CERT_PEM + " http://localhost:80/api/device-connectivity/coaps/certificate/download"); assertThat(linuxCoapCommands.get(COAPS).get(1).asText()).isEqualTo(String.format("coap-client-openssl -v 6 -m POST " + - "-R "+ CA_ROOT_CERT_PEM + " -t \"application/json\" -e \"{temperature:25}\" coaps://test.domain:5684/api/v1/%s/telemetry", credentials.getCredentialsId())); + "-R " + CA_ROOT_CERT_PEM + " -t \"application/json\" -e \"{temperature:25}\" coaps://test.domain:5684/api/v1/%s/telemetry", credentials.getCredentialsId())); JsonNode dockerCoapCommands = commands.get(COAP).get(DOCKER); assertThat(dockerCoapCommands.get(COAP).asText()).isEqualTo(String.format("docker run --rm -it " + diff --git a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java index 41a85721a8..27c7da09b2 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EdgeControllerTest.java @@ -885,6 +885,10 @@ public class EdgeControllerTest extends AbstractControllerTest { device.setType("default"); Device savedDevice = doPost("/api/device", device, Device.class); + // create public customer + doPost("/api/customer/public/device/" + savedDevice.getId().getId(), Device.class); + doDelete("/api/customer/device/" + savedDevice.getId().getId(), Device.class); + simulateEdgeActivation(edge); doPost("/api/edge/" + edge.getId().getId().toString() diff --git a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java index 7d40e56a57..d7aa6f5770 100644 --- a/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/UserControllerTest.java @@ -233,16 +233,9 @@ public class UserControllerTest extends AbstractControllerTest { .put("password", "testPassword2"); Mockito.doNothing().when(mailService).sendPasswordWasResetEmail(anyString(), anyString()); - JsonNode tokenInfo = readResponse( - doPost("/api/noauth/resetPassword", resetPasswordRequest) - .andExpect(status().isOk()), JsonNode.class); + doPost("/api/noauth/resetPassword", resetPasswordRequest) + .andExpect(status().isOk()); Mockito.verify(mailService).sendPasswordWasResetEmail(anyString(), anyString()); - validateAndSetJwtToken(tokenInfo, email); - - doGet("/api/auth/user") - .andExpect(status().isOk()) - .andExpect(jsonPath("$.authority", is(Authority.TENANT_ADMIN.name()))) - .andExpect(jsonPath("$.email", is(email))); resetTokens(); diff --git a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java index feac7adb04..d84e0ebea0 100644 --- a/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/AbstractEdgeTest.java @@ -168,6 +168,10 @@ abstract public class AbstractEdgeTest extends AbstractControllerTest { Device savedDevice = saveDevice("Edge Device 1", THERMOSTAT_DEVICE_PROFILE_NAME); + // create public customer + doPost("/api/customer/public/device/" + savedDevice.getId().getId(), Device.class); + doDelete("/api/customer/device/" + savedDevice.getId().getId(), Device.class); + Asset savedAsset = saveAsset("Edge Asset 1"); updateRootRuleChainMetadata(); diff --git a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java index e58126737d..51cfa44ff9 100644 --- a/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/TenantProfileEdgeTest.java @@ -44,9 +44,8 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { @Test public void testTenantProfiles() throws Exception { loginSysAdmin(); - - // save current values into tmp to revert after test - TenantProfile edgeTenantProfile = doGet("/api/tenantProfile/" + tenantProfileId.getId(), TenantProfile.class); + TenantProfile originalTenantProfile = doGet("/api/tenantProfile/" + tenantProfileId.getId(), TenantProfile.class); + TenantProfile edgeTenantProfile = new TenantProfile(originalTenantProfile); // updated edge tenant profile edgeTenantProfile.setName("Tenant Profile Edge Test"); @@ -64,14 +63,15 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { Assert.assertEquals("Updated tenant profile Edge Test", tenantProfileMsg.getDescription()); Assert.assertEquals("Tenant Profile Edge Test", tenantProfileMsg.getName()); + doPost("/api/tenantProfile", originalTenantProfile, TenantProfile.class); loginTenantAdmin(); } @Test public void testIsolatedTenantProfile() throws Exception { loginSysAdmin(); - - TenantProfile edgeTenantProfile = doGet("/api/tenantProfile/" + tenantProfileId.getId(), TenantProfile.class); + TenantProfile originalTenantProfile = doGet("/api/tenantProfile/" + tenantProfileId.getId(), TenantProfile.class); + TenantProfile edgeTenantProfile = new TenantProfile(originalTenantProfile); // set tenant profile isolated and add 2 queues - main and isolated edgeTenantProfile.setIsolatedTbRuleEngine(true); @@ -110,6 +110,10 @@ public class TenantProfileEdgeTest extends AbstractEdgeTest { Assert.assertNotNull(queue); Assert.assertEquals(tenantId, queue.getTenantId()); } + + loginSysAdmin(); + doPost("/api/tenantProfile", originalTenantProfile, TenantProfile.class); + loginTenantAdmin(); } private TenantProfileQueueConfiguration createQueueConfig(String queueName, String queueTopic) { diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java index 5420cd5297..d478e2099a 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/customer/CustomerService.java @@ -41,6 +41,8 @@ public interface CustomerService extends EntityDaoService { Customer findOrCreatePublicCustomer(TenantId tenantId); + Customer findPublicCustomer(TenantId tenantId); + PageData findCustomersByTenantId(TenantId tenantId, PageLink pageLink); void deleteCustomersByTenantId(TenantId tenantId); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/JavaSerDesUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/JavaSerDesUtil.java index e14e2d5fa1..55a0cd90aa 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/JavaSerDesUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/JavaSerDesUtil.java @@ -36,7 +36,7 @@ public class JavaSerDesUtil { try (ObjectInputStream ois = new ObjectInputStream(is)) { return (T) ois.readObject(); } catch (IOException | ClassNotFoundException e) { - log.error("Error during deserialization message, [{}]", e.getMessage()); + log.error("Error during deserialization", e); return null; } } @@ -50,7 +50,7 @@ public class JavaSerDesUtil { ois.writeObject(msq); return boas.toByteArray(); } catch (IOException e) { - log.error("Error during serialization message, [{}]", e.getMessage()); + log.error("Error during serialization", e); throw new RuntimeException(e); } } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java index 9ba6c20188..77ef4fe5a2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/FieldsUtil.java @@ -289,7 +289,7 @@ public class FieldsUtil { } public static String getText(JsonNode node) { - return node != null ? node.toString() : ""; + return node != null && !node.isNull() ? node.toString() : ""; } private static UUID getCustomerId(CustomerId customerId) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java index 9863506ed4..6f4c7643c0 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edqs/fields/UserFields.java @@ -35,6 +35,11 @@ public class UserFields extends AbstractEntityFields { private String phone; private String additionalInfo; + @Override + public String getName() { + return super.getEmail(); + } + public UserFields(UUID id, long createdTime, UUID tenantId, UUID customerId, Long version, String firstName, String lastName, String email, String phone, JsonNode additionalInfo) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java index 072be6acf2..a1692aef04 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/sync/ie/EntityExportData.java @@ -107,7 +107,7 @@ public class EntityExportData> { @JsonIgnore public boolean hasCalculatedFields() { - return calculatedFields != null; + return calculatedFields != null && !calculatedFields.isEmpty(); } } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java index 634b63e012..cf4267e443 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/CompressedStringDataPoint.java @@ -17,13 +17,12 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import lombok.SneakyThrows; +import org.thingsboard.common.util.TbBytePool; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.edqs.util.TbBytePool; import org.xerial.snappy.Snappy; public class CompressedStringDataPoint extends AbstractDataPoint { - public static final int MIN_STR_SIZE_TO_COMPRESS = 512; @Getter private final byte[] compressedValue; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java index 3a8d570f43..593a74e3c8 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/JsonDataPoint.java @@ -17,7 +17,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.edqs.util.TbStringPool; +import org.thingsboard.common.util.TbStringPool; public class JsonDataPoint extends AbstractDataPoint { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java index 54156500fe..52205e2f72 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/data/dp/StringDataPoint.java @@ -17,7 +17,7 @@ package org.thingsboard.server.edqs.data.dp; import lombok.Getter; import org.thingsboard.server.common.data.kv.DataType; -import org.thingsboard.server.edqs.util.TbStringPool; +import org.thingsboard.common.util.TbStringPool; public class StringDataPoint extends AbstractDataPoint { diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java index 1deaca83a7..215c64194f 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/DefaultEdqsRepository.java @@ -27,10 +27,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.EntityCountQuery; import org.thingsboard.server.common.data.query.EntityDataQuery; -import org.thingsboard.server.edqs.stats.EdqsStatsService; +import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.queue.edqs.EdqsComponent; -import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.function.Predicate; @@ -42,7 +41,7 @@ import java.util.function.Predicate; public class DefaultEdqsRepository implements EdqsRepository { private final static ConcurrentMap repos = new ConcurrentHashMap<>(); - private final Optional statsService; + private final EdqsStatsService statsService; public TenantRepo get(TenantId tenantId) { return repos.computeIfAbsent(tenantId, id -> new TenantRepo(id, statsService)); diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java index b9f3589280..a47559d6d8 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/repo/TenantRepo.java @@ -25,7 +25,6 @@ import org.thingsboard.server.common.data.edqs.EdqsEventType; import org.thingsboard.server.common.data.edqs.EdqsObject; import org.thingsboard.server.common.data.edqs.Entity; import org.thingsboard.server.common.data.edqs.LatestTsKv; -import org.thingsboard.server.common.data.edqs.fields.AssetFields; import org.thingsboard.server.common.data.edqs.fields.EntityFields; import org.thingsboard.server.common.data.edqs.query.QueryResult; import org.thingsboard.server.common.data.id.CustomerId; @@ -41,6 +40,7 @@ import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.edqs.data.ApiUsageStateData; import org.thingsboard.server.edqs.data.AssetData; import org.thingsboard.server.edqs.data.CustomerData; @@ -55,9 +55,7 @@ import org.thingsboard.server.edqs.query.EdqsQuery; import org.thingsboard.server.edqs.query.SortableEntityData; import org.thingsboard.server.edqs.query.processor.EntityQueryProcessor; import org.thingsboard.server.edqs.query.processor.EntityQueryProcessorFactory; -import org.thingsboard.server.edqs.stats.EdqsStatsService; import org.thingsboard.server.edqs.util.RepositoryUtils; -import org.thingsboard.server.edqs.util.TbStringPool; import java.util.ArrayList; import java.util.Collections; @@ -65,7 +63,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.TreeSet; import java.util.UUID; @@ -96,9 +93,9 @@ public class TenantRepo { private final Lock entityUpdateLock = new ReentrantLock(); private final TenantId tenantId; - private final Optional edqsStatsService; + private final EdqsStatsService edqsStatsService; - public TenantRepo(TenantId tenantId, Optional edqsStatsService) { + public TenantRepo(TenantId tenantId, EdqsStatsService edqsStatsService) { this.tenantId = tenantId; this.edqsStatsService = edqsStatsService; } @@ -144,9 +141,9 @@ public class TenantRepo { RelationsRepo repo = relations.computeIfAbsent(entity.getTypeGroup(), tg -> new RelationsRepo()); EntityData from = getOrCreate(entity.getFrom()); EntityData to = getOrCreate(entity.getTo()); - boolean added = repo.add(from, to, TbStringPool.intern(entity.getType())); + boolean added = repo.add(from, to, entity.getType()); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.UPDATED)); + edqsStatsService.reportAdded(ObjectType.RELATION); } } else if (RelationTypeGroup.DASHBOARD.equals(entity.getTypeGroup())) { if (EntityRelation.CONTAINS_TYPE.equals(entity.getType()) && entity.getFrom().getEntityType() == EntityType.CUSTOMER) { @@ -166,7 +163,7 @@ public class TenantRepo { if (relationsRepo != null) { boolean removed = relationsRepo.remove(entityRelation.getFrom().getId(), entityRelation.getTo().getId(), entityRelation.getType()); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.RELATION, EdqsEventType.DELETED)); + edqsStatsService.reportRemoved(ObjectType.RELATION); } } } else if (RelationTypeGroup.DASHBOARD.equals(entityRelation.getTypeGroup())) { @@ -188,7 +185,6 @@ public class TenantRepo { EntityType entityType = entity.getType(); EntityData entityData = getOrCreate(entityType, entityId); - processFields(fields); EntityFields oldFields = entityData.getFields(); entityData.setFields(fields); if (oldFields == null) { @@ -225,7 +221,7 @@ public class TenantRepo { if (removed.getFields() != null) { getEntitySet(entityType).remove(removed); } - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.DELETED)); + edqsStatsService.reportRemoved(entity.type()); UUID customerId = removed.getCustomerId(); if (customerId != null) { @@ -246,7 +242,7 @@ public class TenantRepo { Integer keyId = KeyDictionary.get(attributeKv.getKey()); boolean added = entityData.putAttr(keyId, attributeKv.getScope(), attributeKv.getDataPoint()); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.UPDATED)); + edqsStatsService.reportAdded(ObjectType.ATTRIBUTE_KV); } } } @@ -256,7 +252,7 @@ public class TenantRepo { if (entityData != null) { boolean removed = entityData.removeAttr(KeyDictionary.get(attributeKv.getKey()), attributeKv.getScope()); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.ATTRIBUTE_KV, EdqsEventType.DELETED)); + edqsStatsService.reportRemoved(ObjectType.ATTRIBUTE_KV); } } } @@ -267,7 +263,7 @@ public class TenantRepo { Integer keyId = KeyDictionary.get(latestTsKv.getKey()); boolean added = entityData.putTs(keyId, latestTsKv.getDataPoint()); if (added) { - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.UPDATED)); + edqsStatsService.reportAdded(ObjectType.LATEST_TS_KV); } } } @@ -277,17 +273,11 @@ public class TenantRepo { if (entityData != null) { boolean removed = entityData.removeTs(KeyDictionary.get(latestTsKv.getKey())); if (removed) { - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.LATEST_TS_KV, EdqsEventType.DELETED)); + edqsStatsService.reportRemoved(ObjectType.LATEST_TS_KV); } } } - public void processFields(EntityFields fields) { - if (fields instanceof AssetFields assetFields) { - assetFields.setType(TbStringPool.intern(assetFields.getType())); - } - } - public ConcurrentMap> getEntityMap(EntityType entityType) { return entityMapByType.computeIfAbsent(entityType, et -> new ConcurrentHashMap<>()); } @@ -301,7 +291,7 @@ public class TenantRepo { return getEntityMap(entityType).computeIfAbsent(entityId, id -> { log.debug("[{}] Adding {} {}", tenantId, entityType, id); EntityData entityData = constructEntityData(entityType, entityId); - edqsStatsService.ifPresent(statService -> statService.reportEvent(tenantId, ObjectType.fromEntityType(entityType), EdqsEventType.UPDATED)); + edqsStatsService.reportAdded(ObjectType.fromEntityType(entityType)); return entityData; }); } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java new file mode 100644 index 0000000000..3767d0f60b --- /dev/null +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/DefaultEdqsStatsService.java @@ -0,0 +1,92 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edqs.stats; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; +import org.thingsboard.server.common.stats.EdqsStatsService; +import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.common.stats.StatsTimer; +import org.thingsboard.server.common.stats.StatsType; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Service +@Slf4j +@ConditionalOnExpression("'${queue.edqs.api.supported:true}' == 'true' && '${queue.edqs.stats.enabled:true}' == 'true'") +public class DefaultEdqsStatsService implements EdqsStatsService { + + private final StatsFactory statsFactory; + + @Value("${queue.edqs.stats.slow_query_threshold:3000}") + private int slowQueryThreshold; + + private final ConcurrentHashMap objectCounters = new ConcurrentHashMap<>(); + private final StatsTimer dataQueryTimer; + private final StatsTimer countQueryTimer; + + private DefaultEdqsStatsService(StatsFactory statsFactory) { + this.statsFactory = statsFactory; + dataQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityDataQueryTimer"); + countQueryTimer = statsFactory.createTimer(StatsType.EDQS, "entityCountQueryTimer"); + } + + @Override + public void reportAdded(ObjectType objectType) { + getObjectCounter(objectType).incrementAndGet(); + } + + @Override + public void reportRemoved(ObjectType objectType) { + getObjectCounter(objectType).decrementAndGet(); + } + + @Override + public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) { + double timingMs = timingNanos / 1000_000.0; + if (timingMs < slowQueryThreshold) { + log.debug("[{}] Executed data query in {} ms: {}", tenantId, timingMs, query); + } else { + log.warn("[{}] Executed slow data query in {} ms: {}", tenantId, timingMs, query); + } + dataQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS); + } + + @Override + public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) { + double timingMs = timingNanos / 1000_000.0; + if (timingMs < slowQueryThreshold) { + log.debug("[{}] Executed count query in {} ms: {}", tenantId, timingMs, query); + } else { + log.warn("[{}] Executed slow count query in {} ms: {}", tenantId, timingMs, query); + } + countQueryTimer.record(timingNanos, TimeUnit.NANOSECONDS); + } + + private AtomicInteger getObjectCounter(ObjectType objectType) { + return objectCounters.computeIfAbsent(objectType, type -> + statsFactory.createGauge("edqsObjectsCount", new AtomicInteger(), "objectType", type.name())); + } + +} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java deleted file mode 100644 index 442453fc93..0000000000 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/stats/EdqsStatsService.java +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Copyright © 2016-2025 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.edqs.stats; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.ObjectType; -import org.thingsboard.server.common.data.edqs.EdqsEventType; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.common.stats.StatsType; -import org.thingsboard.server.queue.edqs.EdqsComponent; - -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -@EdqsComponent -@Service -@Slf4j -@RequiredArgsConstructor -@ConditionalOnProperty(name = "queue.edqs.stats.enabled", havingValue = "true", matchIfMissing = true) -public class EdqsStatsService { - - private final ConcurrentHashMap statsMap = new ConcurrentHashMap<>(); - private final StatsFactory statsFactory; - - public void reportEvent(TenantId tenantId, ObjectType objectType, EdqsEventType eventType) { - statsMap.computeIfAbsent(tenantId, id -> new EdqsStats(tenantId, statsFactory)) - .reportEvent(objectType, eventType); - } - - @Getter - @AllArgsConstructor - static class EdqsStats { - - private final TenantId tenantId; - private final ConcurrentHashMap entityCounters = new ConcurrentHashMap<>(); - private final StatsFactory statsFactory; - - private AtomicInteger getOrCreateObjectCounter(ObjectType objectType) { - return entityCounters.computeIfAbsent(objectType, - type -> statsFactory.createGauge(StatsType.EDQS.getName() + "_object_count", new AtomicInteger(), - "tenantId", tenantId.toString(), "objectType", type.name())); - } - - @Override - public String toString() { - return entityCounters.entrySet().stream() - .map(counters -> counters.getKey().name()+ " total = [" + counters.getValue() + "]") - .collect(Collectors.joining(", ")); - } - - public void reportEvent(ObjectType objectType, EdqsEventType eventType) { - AtomicInteger objectCounter = getOrCreateObjectCounter(objectType); - if (eventType == EdqsEventType.UPDATED){ - objectCounter.incrementAndGet(); - } else if (eventType == EdqsEventType.DELETED) { - objectCounter.decrementAndGet(); - } - } - } - -} diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java index 5b4cd7ac4a..167037b889 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java +++ b/common/edqs/src/main/java/org/thingsboard/server/edqs/util/EdqsConverter.java @@ -17,13 +17,19 @@ package org.thingsboard.server.edqs.util; import com.fasterxml.jackson.annotation.JsonAutoDetect; import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.json.JsonMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import com.google.protobuf.ByteString; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.TbStringPool; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.ObjectType; @@ -49,6 +55,7 @@ import org.thingsboard.server.gen.transport.TransportProtos; import org.thingsboard.server.gen.transport.TransportProtos.DataPointProto; import org.xerial.snappy.Snappy; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -57,6 +64,9 @@ import java.util.UUID; @Slf4j public class EdqsConverter { + @Value("${queue.edqs.string_compression_length_threshold:512}") + private int stringCompressionLengthThreshold; + private final Map> converters = new HashMap<>(); private final Converter defaultConverter = new JsonConverter<>(Entity.class); @@ -125,7 +135,7 @@ public class EdqsConverter { }); } - public static DataPointProto toDataPointProto(long ts, KvEntry kvEntry) { + public DataPointProto toDataPointProto(long ts, KvEntry kvEntry) { DataPointProto.Builder proto = DataPointProto.newBuilder(); proto.setTs(ts); switch (kvEntry.getDataType()) { @@ -134,7 +144,7 @@ public class EdqsConverter { case DOUBLE -> proto.setDoubleV(kvEntry.getDoubleValue().get()); case STRING -> { String strValue = kvEntry.getStrValue().get(); - if (strValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + if (strValue.length() < stringCompressionLengthThreshold) { proto.setStringV(strValue); } else { proto.setCompressedStringV(ByteString.copyFrom(compress(strValue))); @@ -142,7 +152,7 @@ public class EdqsConverter { } case JSON -> { String jsonValue = kvEntry.getJsonValue().get(); - if (jsonValue.length() < CompressedStringDataPoint.MIN_STR_SIZE_TO_COMPRESS) { + if (jsonValue.length() < stringCompressionLengthThreshold) { proto.setJsonV(jsonValue); } else { proto.setCompressedJsonV(ByteString.copyFrom(compress(jsonValue))); @@ -152,7 +162,7 @@ public class EdqsConverter { return proto.build(); } - public static DataPoint fromDataPointProto(DataPointProto proto) { + public DataPoint fromDataPointProto(DataPointProto proto) { long ts = proto.getTs(); if (proto.hasBoolV()) { return new BoolDataPoint(ts, proto.getBoolV()); @@ -188,14 +198,6 @@ public class EdqsConverter { return edqsEntity; } - public EdqsObject check(ObjectType type, Object object) { - if (object instanceof EdqsObject edqsObject) { - return edqsObject; - } else { - return toEntity(type.toEntityType(), object); - } - } - @SuppressWarnings("unchecked") @SneakyThrows public byte[] serialize(ObjectType type, T value) { @@ -220,12 +222,18 @@ public class EdqsConverter { @RequiredArgsConstructor private static class JsonConverter implements Converter { + private static final SimpleModule module = new SimpleModule(); private static final ObjectMapper mapper = JsonMapper.builder() .visibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY) .visibility(PropertyAccessor.GETTER, JsonAutoDetect.Visibility.NONE) .visibility(PropertyAccessor.IS_GETTER, JsonAutoDetect.Visibility.NONE) .build(); + static { + module.addDeserializer(String.class, new InterningStringDeserializer()); + mapper.registerModule(module); + } + private final Class type; @SneakyThrows @@ -250,4 +258,17 @@ public class EdqsConverter { } + public static class InterningStringDeserializer extends StdDeserializer { + + public InterningStringDeserializer() { + super(String.class); + } + + @Override + public String deserialize(JsonParser p, DeserializationContext ctx) throws IOException { + return TbStringPool.intern(p.getText()); + } + + } + } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java index 42263138c1..7271e1cb10 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/TopicService.java @@ -38,7 +38,7 @@ public class TopicService { @Value("${queue.rule-engine.notifications-topic:tb_rule_engine.notifications}") private String tbRuleEngineNotificationsTopic; - @Value("${queue.transport.notifications-topics:tb_transport.notifications}") + @Value("${queue.transport.notifications-topic:tb_transport.notifications}") private String tbTransportNotificationsTopic; @Value("${queue.edge.notifications-topic:tb_edge.notifications}") diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java index 181d422f16..82b5af179f 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/kafka/TbKafkaSettings.java @@ -50,8 +50,6 @@ import java.util.Properties; @Component public class TbKafkaSettings { - private static final List DYNAMIC_TOPICS = List.of("tb_edge_event.notifications"); - @Value("${queue.kafka.bootstrap.servers}") private String servers; @@ -163,18 +161,20 @@ public class TbKafkaSettings { props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class); props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class); - consumerPropertiesPerTopic - .getOrDefault(topic, Collections.emptyList()) - .forEach(kv -> props.put(kv.getKey(), kv.getValue())); - if (topic != null) { - DYNAMIC_TOPICS.stream() - .filter(topic::startsWith) - .findFirst() - .ifPresent(prefix -> consumerPropertiesPerTopic.getOrDefault(prefix, Collections.emptyList()) - .forEach(kv -> props.put(kv.getKey(), kv.getValue()))); + List properties = consumerPropertiesPerTopic.get(topic); + if (properties == null) { + for (Map.Entry> entry : consumerPropertiesPerTopic.entrySet()) { + if (topic.startsWith(entry.getKey())) { + properties = entry.getValue(); + break; + } + } + } + if (properties != null) { + properties.forEach(kv -> props.put(kv.getKey(), kv.getValue())); + } } - return props; } diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java index 7c3e415e9f..98a3d78304 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/provider/TbCoreQueueProducerProvider.java @@ -115,7 +115,6 @@ public class TbCoreQueueProducerProvider implements TbQueueProducerProvider { return toHousekeeper; } - @Override public TbQueueProducer> getTbEdgeMsgProducer() { return toEdge; diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java new file mode 100644 index 0000000000..df78e5fc89 --- /dev/null +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/DummyEdqsStatsService.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.stats; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +@Service +@ConditionalOnMissingBean(value = EdqsStatsService.class, ignored = DummyEdqsStatsService.class) +public class DummyEdqsStatsService implements EdqsStatsService { + + @Override + public void reportAdded(ObjectType objectType) {} + + @Override + public void reportRemoved(ObjectType objectType) {} + + @Override + public void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos) {} + + @Override + public void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos) {} + +} diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java new file mode 100644 index 0000000000..106e43e913 --- /dev/null +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/EdqsStatsService.java @@ -0,0 +1,33 @@ +/** + * Copyright © 2016-2025 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.common.stats; + +import org.thingsboard.server.common.data.ObjectType; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.query.EntityCountQuery; +import org.thingsboard.server.common.data.query.EntityDataQuery; + +public interface EdqsStatsService { + + void reportAdded(ObjectType objectType); + + void reportRemoved(ObjectType objectType); + + void reportDataQuery(TenantId tenantId, EntityDataQuery query, long timingNanos); + + void reportCountQuery(TenantId tenantId, EntityCountQuery query, long timingNanos); + +} diff --git a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java index 89f33ac922..b9f3533b92 100644 --- a/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java +++ b/common/stats/src/main/java/org/thingsboard/server/common/stats/StatsTimer.java @@ -34,10 +34,14 @@ public class StatsTimer { this.timer = micrometerTimer; } - public void record(long timeMs) { + public void record(long timeMs) { + record(timeMs, TimeUnit.MILLISECONDS); + } + + public void record(long timing, TimeUnit timeUnit) { count++; - totalTime += timeMs; - timer.record(timeMs, TimeUnit.MILLISECONDS); + totalTime += timeUnit.toMillis(timing); + timer.record(timing, timeUnit); } public double getAvg() { @@ -47,7 +51,7 @@ public class StatsTimer { return (double) totalTime / count; } - public void reset() { + public void reset() { count = 0; totalTime = 0; } diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java b/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java similarity index 96% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java rename to common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java index 3b135be59c..fe16a14e7c 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbBytePool.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbBytePool.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs.util; +package org.thingsboard.common.util; import com.google.common.hash.Hashing; import org.springframework.util.ConcurrentReferenceHashMap; diff --git a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java b/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java similarity index 96% rename from common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java rename to common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java index 9c9c3b5b13..38c010fbd3 100644 --- a/common/edqs/src/main/java/org/thingsboard/server/edqs/util/TbStringPool.java +++ b/common/util/src/main/java/org/thingsboard/common/util/TbStringPool.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.edqs.util; +package org.thingsboard.common.util; import org.springframework.util.ConcurrentReferenceHashMap; diff --git a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java index 3500e48a79..df0f79171f 100644 --- a/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java +++ b/common/version-control/src/main/java/org/thingsboard/server/service/sync/vc/GitRepository.java @@ -42,7 +42,6 @@ import org.eclipse.jgit.diff.RawText; import org.eclipse.jgit.diff.RawTextComparator; import org.eclipse.jgit.errors.LargeObjectException; import org.eclipse.jgit.errors.RepositoryNotFoundException; -import org.eclipse.jgit.errors.TransportException; import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ObjectLoader; @@ -158,7 +157,7 @@ public class GitRepository { return repository; } catch (RepositoryNotFoundException e) { log.warn("{} not a git repository, reinitializing", directory); - } catch (TransportException e) { + } catch (org.eclipse.jgit.errors.TransportException | org.eclipse.jgit.api.errors.TransportException e) { if (StringUtils.containsIgnoreCase(e.getMessage(), "missing commit")) { log.warn("Couldn't fetch {} due to {}, reinitializing", directory, e.getMessage()); } else { diff --git a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java index 4b05cdbc75..b06902d95e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/customer/CustomerServiceImpl.java @@ -213,12 +213,11 @@ public class CustomerServiceImpl extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); - Optional publicCustomerOpt = customerDao.findPublicCustomerByTenantId(tenantId.getId()); - if (publicCustomerOpt.isPresent()) { - return publicCustomerOpt.get(); + var publicCustomer = findPublicCustomer(tenantId); + if (publicCustomer != null) { + return publicCustomer; } - var publicCustomer = new Customer(); + publicCustomer = new Customer(); publicCustomer.setTenantId(tenantId); publicCustomer.setTitle(PUBLIC_CUSTOMER_TITLE); try { @@ -230,7 +229,7 @@ public class CustomerServiceImpl extends AbstractCachedEntityService publicCustomerOpt = customerDao.findPublicCustomerByTenantId(tenantId.getId()); if (publicCustomerOpt.isPresent()) { return publicCustomerOpt.get(); } @@ -239,6 +238,14 @@ public class CustomerServiceImpl extends AbstractCachedEntityService INCORRECT_TENANT_ID + id); + Optional publicCustomerOpt = customerDao.findPublicCustomerByTenantId(tenantId.getId()); + return publicCustomerOpt.orElse(null); + } + @Override public PageData findCustomersByTenantId(TenantId tenantId, PageLink pageLink) { log.trace("Executing findCustomersByTenantId, tenantId [{}], pageLink [{}]", tenantId, pageLink); diff --git a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java index eb7ce56b9f..6f22e95c87 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/device/DeviceConnectivityServiceImpl.java @@ -85,7 +85,7 @@ public class DeviceConnectivityServiceImpl implements DeviceConnectivityService private String mqttsPemCertFile; @Value("${device.connectivity.coaps.pem_cert_file:}") private String coapsPemCertFile; - @Value("${device.connectivity.gateway.image_version:latest}") + @Value("${device.connectivity.gateway.image_version:3.7-stable}") private String gatewayImageVersion; @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java index a762aabf38..4df33779ee 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/entity/BaseEntityService.java @@ -20,6 +20,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; +import org.thingsboard.common.util.TbStopWatch; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.HasCustomerId; import org.thingsboard.server.common.data.HasEmail; @@ -46,6 +47,7 @@ import org.thingsboard.server.common.data.query.EntityTypeFilter; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.RelationsQueryFilter; import org.thingsboard.server.common.msg.edqs.EdqsApiService; +import org.thingsboard.server.common.stats.EdqsStatsService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import java.util.ArrayList; @@ -89,6 +91,9 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe @Lazy private EdqsApiService edqsApiService; + @Autowired + private EdqsStatsService edqsStatsService; + @Override public long countEntitiesByQuery(TenantId tenantId, CustomerId customerId, EntityCountQuery query) { log.trace("Executing countEntitiesByQuery, tenantId [{}], customerId [{}], query [{}]", tenantId, customerId, query); @@ -96,14 +101,19 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityCountQuery(query); + TbStopWatch stopWatch = TbStopWatch.create(); + Long result; if (edqsApiService.isEnabled() && validForEdqs(query) && !tenantId.isSysTenantId()) { EdqsRequest request = EdqsRequest.builder() .entityCountQuery(query) .build(); EdqsResponse response = processEdqsRequest(tenantId, customerId, request); - return response.getEntityCountQueryResult(); + result = response.getEntityCountQueryResult(); + } else { + result = entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); } - return this.entityQueryDao.countEntitiesByQuery(tenantId, customerId, query); + edqsStatsService.reportCountQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos()); + return result; } @Override @@ -113,27 +123,31 @@ public class BaseEntityService extends AbstractEntityService implements EntitySe validateId(customerId, id -> INCORRECT_CUSTOMER_ID + id); validateEntityDataQuery(query); + TbStopWatch stopWatch = TbStopWatch.create(); + PageData result; if (edqsApiService.isEnabled() && validForEdqs(query)) { EdqsRequest request = EdqsRequest.builder() .entityDataQuery(query) .build(); EdqsResponse response = processEdqsRequest(tenantId, customerId, request); - return response.getEntityDataQueryResult(); - } - - if (!isValidForOptimization(query)) { - return this.entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); - } - - // 1 step - find entity data by filter and sort columns - PageData entityDataByQuery = findEntityIdsByFilterAndSorterColumns(tenantId, customerId, query); - if (entityDataByQuery == null || entityDataByQuery.getData().isEmpty()) { - return entityDataByQuery; + result = response.getEntityDataQueryResult(); + } else { + if (!isValidForOptimization(query)) { + result = entityQueryDao.findEntityDataByQuery(tenantId, customerId, query); + } else { + // 1 step - find entity data by filter and sort columns + PageData entityDataByQuery = findEntityIdsByFilterAndSorterColumns(tenantId, customerId, query); + if (entityDataByQuery == null || entityDataByQuery.getData().isEmpty()) { + result = entityDataByQuery; + } else { + // 2 step - find entity data by entity ids from the 1st step + List entities = fetchEntityDataByIdsFromInitialQuery(tenantId, customerId, query, entityDataByQuery.getData()); + result = new PageData<>(entities, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); + } + } } - - // 2 step - find entity data by entity ids from the 1st step - List result = fetchEntityDataByIdsFromInitialQuery(tenantId, customerId, query, entityDataByQuery.getData()); - return new PageData<>(result, entityDataByQuery.getTotalPages(), entityDataByQuery.getTotalElements(), entityDataByQuery.hasNext()); + edqsStatsService.reportDataQuery(tenantId, query, stopWatch.stopAndGetTotalTimeNanos()); + return result; } private boolean validForEdqs(EntityCountQuery query) { // for compatibility with PE diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 353d76391b..2c0e0b8c5d 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -71,6 +71,8 @@ queue: max_pending_requests: "${TB_EDQS_MAX_PENDING_REQUESTS:10000}" # Maximum timeout for requests to EDQS max_request_timeout: "${TB_EDQS_MAX_REQUEST_TIMEOUT:20000}" + # Strings longer than this threshold will be compressed + string_compression_length_threshold: "${TB_EDQS_STRING_COMPRESSION_LENGTH_THRESHOLD:512}" stats: # Enable/disable statistics for EDQS enabled: "${TB_EDQS_STATS_ENABLED:true}" diff --git a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java index 01a7495148..330d22a3c6 100644 --- a/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java +++ b/edqs/src/test/java/org/thingsboard/server/edqs/repo/AbstractEDQTest.java @@ -20,6 +20,7 @@ import org.junit.Before; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.domain.EntityScan; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.test.context.TestExecutionListeners; @@ -58,6 +59,7 @@ import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.StringFilterPredicate; import org.thingsboard.server.common.data.relation.EntityRelation; import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.stats.DummyEdqsStatsService; import org.thingsboard.server.edqs.util.EdqsConverter; import java.util.Collections; @@ -78,6 +80,8 @@ public abstract class AbstractEDQTest { protected DefaultEdqsRepository repository; @Autowired protected EdqsConverter edqsConverter; + @MockBean + private DummyEdqsStatsService edqsStatsService; protected final TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); protected final CustomerId customerId = new CustomerId(UUID.randomUUID()); diff --git a/ui-ngx/src/app/core/auth/auth.models.ts b/ui-ngx/src/app/core/auth/auth.models.ts index 1944c693ac..e5cc1424ab 100644 --- a/ui-ngx/src/app/core/auth/auth.models.ts +++ b/ui-ngx/src/app/core/auth/auth.models.ts @@ -31,6 +31,7 @@ export interface SysParamsState { maxDataPointsPerRollingArg: number; maxArgumentsPerCF: number; ruleChainDebugPerTenantLimitsConfiguration?: string; + calculatedFieldDebugPerTenantLimitsConfiguration?: string; } export interface SysParams extends SysParamsState { diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index ba0161ed02..f8ffd5e8b6 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -152,12 +152,8 @@ export class AuthService { )); } - public resetPassword(resetToken: string, password: string): Observable { - return this.http.post('/api/noauth/resetPassword', {resetToken, password}, defaultHttpOptions()).pipe( - tap((loginResponse: LoginResponse) => { - this.setUserFromJwtToken(loginResponse.token, loginResponse.refreshToken, true); - } - )); + public resetPassword(resetToken: string, password: string): Observable { + return this.http.post('/api/noauth/resetPassword', {resetToken, password}, defaultHttpOptions()); } public changePassword(currentPassword: string, newPassword: string, config?: RequestConfig) { diff --git a/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts b/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts index fb0ff6de90..b450940ca1 100644 --- a/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/alarm/alarm-table-config.ts @@ -243,7 +243,7 @@ export class AlarmTableConfig extends EntityTableConfig if ($event) { $event.stopPropagation(); } - const target = $event.target || $event.srcElement || $event.currentTarget; + const target = $event.target || $event.currentTarget; const config = new OverlayConfig(); config.backdropClass = 'cdk-overlay-transparent-backdrop'; config.hasBackdrop = true; diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts index 538afa3a06..d4ff2dc041 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.ts @@ -338,7 +338,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI if (this.isClientSideTelemetryTypeMap.get(this.attributeScope)) { return; } - const target = $event.target || $event.srcElement || $event.currentTarget; + const target = $event.target || $event.currentTarget; const config = new OverlayConfig(); config.backdropClass = 'cdk-overlay-transparent-backdrop'; config.hasBackdrop = true; @@ -389,7 +389,7 @@ export class AttributeTableComponent extends PageComponent implements AfterViewI if ($event) { $event.stopPropagation(); } - const target = $event.target || $event.srcElement || $event.currentTarget; + const target = $event.target || $event.currentTarget; const config = new OverlayConfig({ panelClass: 'tb-filter-panel', backdropClass: 'cdk-overlay-transparent-backdrop', diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts index c5b256ea89..07f8f0293c 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/calculated-fields-table-config.ts @@ -27,10 +27,9 @@ import { PageLink } from '@shared/models/page/page-link'; import { Observable, of } from 'rxjs'; import { PageData } from '@shared/models/page/page-data'; import { EntityId } from '@shared/models/id/entity-id'; -import { MINUTE } from '@shared/models/time/time.models'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; -import { getCurrentAuthState, getCurrentAuthUser } from '@core/auth/auth.selectors'; +import { getCurrentAuthUser } from '@core/auth/auth.selectors'; import { DestroyRef, Renderer2 } from '@angular/core'; import { EntityDebugSettings } from '@shared/models/entity.models'; import { takeUntilDestroyed } from '@angular/core/rxjs-interop'; @@ -47,7 +46,8 @@ import { getCalculatedFieldArgumentsHighlights, } from '@shared/models/calculated-field.models'; import { - CalculatedFieldDebugDialogComponent, CalculatedFieldDebugDialogData, + CalculatedFieldDebugDialogComponent, + CalculatedFieldDebugDialogData, CalculatedFieldDialogComponent, CalculatedFieldDialogData, CalculatedFieldScriptTestDialogComponent, @@ -60,9 +60,6 @@ import { DatePipe } from '@angular/common'; export class CalculatedFieldsTableConfig extends EntityTableConfig { - readonly calculatedFieldsDebugPerTenantLimitsConfiguration = - getCurrentAuthState(this.store)['calculatedFieldsDebugPerTenantLimitsConfiguration']; - readonly maxDebugModeDuration = getCurrentAuthState(this.store).maxDebugModeDurationMinutes * MINUTE; readonly tenantId = getCurrentAuthUser(this.store).tenantId; additionalDebugActionConfig = { title: this.translate.instant('calculated-fields.see-debug-events'), @@ -189,9 +186,7 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig this.onDebugConfigChanged(id.id, settings) @@ -215,7 +210,6 @@ export class CalculatedFieldsTableConfig extends EntityTableConfig diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts index 0fcac393f0..478447d1d9 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/debug-dialog/calculated-field-debug-dialog.component.ts @@ -55,8 +55,8 @@ export class CalculatedFieldDebugDialogComponent extends DialogComponent this.data.value.type === CalculatedFieldType.SCRIPT && !!(event as Event).body.arguments); this.eventsTable.entitiesTable.updateData(); - this.eventsTable.entitiesTable.cellActionDescriptors[0].isEnabled = (event => this.data.value.type === CalculatedFieldType.SCRIPT && !!(event as Event).body.arguments) } cancel(): void { diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html index 1688685523..8e9baa6f90 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.html @@ -49,8 +49,7 @@ @@ -80,6 +79,12 @@ +
+
@if (configFormGroup.get('expressionSIMPLE').errors && configFormGroup.get('expressionSIMPLE').touched) { @if (configFormGroup.get('expressionSIMPLE').hasError('required')) { @@ -110,7 +115,7 @@
{{ 'api-usage.tbel' | translate }}
diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts index 169d6dff50..52051aa6f7 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/dialog/calculated-field-dialog.component.ts @@ -48,7 +48,6 @@ export interface CalculatedFieldDialogData { value?: CalculatedField; buttonTitle: string; entityId: EntityId; - debugLimitsConfiguration: string; tenantId: string; entityName?: string; additionalDebugActionConfig: AdditionalDebugActionConfig<(calculatedField: CalculatedField) => void>; diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html index 7521395cb4..92855d882f 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.html @@ -55,11 +55,11 @@ class="tb-error"> warning - } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('equalCtx')) { + } @else if (argumentFormGroup.get('argumentName').touched && argumentFormGroup.get('argumentName').hasError('forbiddenName')) { warning diff --git a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts index 2a64c90f20..6f46e47af9 100644 --- a/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts +++ b/ui-ngx/src/app/modules/home/components/calculated-fields/components/panel/calculated-field-argument-panel.component.ts @@ -67,7 +67,7 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI readonly defaultLimit = Math.floor(this.maxDataPointsPerRollingArg / 10); argumentFormGroup = this.fb.group({ - argumentName: ['', [Validators.required, this.uniqNameRequired(), this.notEqualCtxValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], + argumentName: ['', [Validators.required, this.uniqNameRequired(), this.forbiddenArgumentNameValidator(), Validators.pattern(charsWithNumRegex), Validators.maxLength(255)]], refEntityId: this.fb.group({ entityType: [ArgumentEntityType.Current], id: [''] @@ -254,10 +254,11 @@ export class CalculatedFieldArgumentPanelComponent implements OnInit, AfterViewI } } - private notEqualCtxValidator(): ValidatorFn { + private forbiddenArgumentNameValidator(): ValidatorFn { return (control: FormControl) => { const trimmedValue = control.value.trim().toLowerCase(); - return trimmedValue === 'ctx' ? { equalCtx: true } : null; + const forbiddenArgumentNames = ['ctx', 'e', 'pi']; + return forbiddenArgumentNames.includes(trimmedValue) ? { forbiddenName: true } : null; }; } diff --git a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html index 5fd247e072..77f882d191 100644 --- a/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html +++ b/ui-ngx/src/app/modules/home/components/dashboard-page/dashboard-page.component.html @@ -365,7 +365,7 @@ close
- Powered by Thingsboard v.{{ thingsboardVersion }} + Powered by ThingsBoard v.{{ thingsboardVersion }}
{ + this.widgetComponentService.getWidgetInfo(widget.typeFullFqn).subscribe({ + next: (widgetTypeInfo) => { const config: WidgetConfig = this.dashboardUtils.widgetConfigFromWidgetType(widgetTypeInfo); if (!config.title) { config.title = 'New ' + widgetTypeInfo.widgetName; @@ -1389,8 +1389,13 @@ export class DashboardPageComponent extends PageComponent implements IDashboardC } }); } + }, + error: (errorData) => { + const errorMessages: string[] = errorData.errorMessages; + this.dialogService.alert(this.translate.instant('widget.widget-type-load-error'), + errorMessages.join('
').replace(/\n/g, '
')); } - ); + }); } onRevertWidgetEdit() { diff --git a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.html b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.html index 4243044fc8..f79abfe904 100644 --- a/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.html +++ b/ui-ngx/src/app/modules/home/components/entity/debug/entity-debug-settings-button.component.html @@ -16,6 +16,7 @@ -->
-
+
rulenode.name @@ -38,8 +38,7 @@