diff --git a/.gitignore b/.gitignore index 777a52efc7..6c5003fc02 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,4 @@ rebuild-docker.sh */.run/** .run/** .run -.claude/ +.claude diff --git a/application/src/main/data/upgrade/basic/schema_update.sql b/application/src/main/data/upgrade/basic/schema_update.sql index a2dfeac358..8b53eda2bf 100644 --- a/application/src/main/data/upgrade/basic/schema_update.sql +++ b/application/src/main/data/upgrade/basic/schema_update.sql @@ -14,3 +14,8 @@ -- limitations under the License. -- +-- CALCULATED FIELD ADDITIONAL INFO ADDITION START + +ALTER TABLE calculated_field ADD COLUMN IF NOT EXISTS additional_info varchar; + +-- CALCULATED FIELD ADDITIONAL INFO ADDITION END diff --git a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java index 38f37cf35d..4ce7268232 100644 --- a/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/WebSocketConfiguration.java @@ -60,7 +60,8 @@ public class WebSocketConfiguration implements WebSocketConfigurer { log.error("TbWebSocketHandler expected but [{}] provided", wsHandler); throw new RuntimeException("TbWebSocketHandler expected but " + wsHandler + " provided"); } - registry.addHandler(wsHandler, WS_API_MAPPING).setAllowedOriginPatterns("*"); + registry.addHandler(wsHandler, WS_API_MAPPING) + .setAllowedOriginPatterns("*"); } } diff --git a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java index 73315be73f..48d3a907bc 100644 --- a/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java +++ b/application/src/main/java/org/thingsboard/server/controller/plugin/TbWebSocketHandler.java @@ -52,6 +52,7 @@ import org.thingsboard.server.config.WebSocketConfiguration; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; +import org.thingsboard.server.service.security.auth.pat.ApiKeyAuthenticationProvider; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -100,6 +101,8 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke private RateLimitService rateLimitService; @Autowired private JwtAuthenticationProvider authenticationProvider; + @Autowired + private ApiKeyAuthenticationProvider apiKeyAuthenticationProvider; @Value("${server.ws.send_timeout:5000}") private long sendTimeout; @@ -194,7 +197,11 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke log.trace("{} Authenticating session", sessionRef); SecurityUser securityCtx; try { - securityCtx = authenticationProvider.authenticate(authCmd.getToken()); + if (StringUtils.isNotEmpty(authCmd.getApiKey())) { + securityCtx = apiKeyAuthenticationProvider.authenticate(authCmd.getApiKey()); + } else { + securityCtx = authenticationProvider.authenticate(authCmd.getToken()); + } } catch (Exception e) { close(sessionRef, CloseStatus.BAD_DATA.withReason(e.getMessage())); return; @@ -328,9 +335,12 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke } SecurityUser securityCtx = null; - String token = StringUtils.substringAfter(session.getUri().getQuery(), "token="); - if (StringUtils.isNotEmpty(token)) { - securityCtx = authenticationProvider.authenticate(token); + String query = session.getUri().getQuery(); + if (query != null) { + String token = extractQueryParam(query, "token"); + if (StringUtils.isNotEmpty(token)) { + securityCtx = authenticationProvider.authenticate(token); + } } return WebSocketSessionRef.builder() .sessionId(UUID.randomUUID().toString()) @@ -341,6 +351,15 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke .build(); } + private String extractQueryParam(String query, String paramName) { + for (String param : query.split("&")) { + if (param.startsWith(paramName + "=")) { + return param.substring(paramName.length() + 1); + } + } + return null; + } + private SessionMetaData getSessionMd(String internalSessionId) { SessionMetaData sessionMd = internalSessionMap.get(internalSessionId); if (sessionMd == null) { @@ -482,6 +501,7 @@ public class TbWebSocketHandler extends TextWebSocketHandler implements WebSocke } } } + } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java index c03c6affe2..c99261b249 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeContextComponent.java @@ -25,6 +25,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.edge.EdgeEventType; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.dao.ai.AiModelService; +import org.thingsboard.server.dao.pat.ApiKeyService; import org.thingsboard.server.dao.alarm.AlarmCommentService; import org.thingsboard.server.dao.alarm.AlarmService; import org.thingsboard.server.dao.asset.AssetProfileService; @@ -61,6 +62,7 @@ import org.thingsboard.server.service.edge.rpc.EdgeEventStorageSettings; import org.thingsboard.server.service.edge.rpc.EdgeRpcService; import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; import org.thingsboard.server.service.edge.rpc.processor.ai.AiModelProcessor; +import org.thingsboard.server.service.edge.rpc.processor.apikey.ApiKeyProcessor; import org.thingsboard.server.service.edge.rpc.processor.alarm.AlarmProcessor; import org.thingsboard.server.service.edge.rpc.processor.alarm.comment.AlarmCommentProcessor; import org.thingsboard.server.service.edge.rpc.processor.asset.AssetEdgeProcessor; @@ -273,6 +275,12 @@ public class EdgeContextComponent { @Autowired private AiModelProcessor aiModelProcessor; + @Autowired + private ApiKeyService apiKeyService; + + @Autowired + private ApiKeyProcessor apiKeyProcessor; + @Autowired private UserProcessor userProcessor; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java index 445741fd50..01d2178cd1 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/EdgeMsgConstructorUtils.java @@ -49,6 +49,7 @@ import org.thingsboard.server.common.data.Tenant; import org.thingsboard.server.common.data.TenantProfile; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.ai.AiModel; +import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.alarm.Alarm; import org.thingsboard.server.common.data.alarm.AlarmComment; import org.thingsboard.server.common.data.asset.Asset; @@ -61,6 +62,7 @@ import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.id.AiModelId; +import org.thingsboard.server.common.data.id.ApiKeyId; import org.thingsboard.server.common.data.id.AssetId; import org.thingsboard.server.common.data.id.AssetProfileId; import org.thingsboard.server.common.data.id.CalculatedFieldId; @@ -98,6 +100,7 @@ import org.thingsboard.server.common.data.widget.WidgetTypeDetails; import org.thingsboard.server.common.data.widget.WidgetsBundle; import org.thingsboard.server.common.transport.util.JsonUtils; import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; @@ -741,6 +744,19 @@ public class EdgeMsgConstructorUtils { .setIdLSB(aiModelId.getId().getLeastSignificantBits()).build(); } + public static ApiKeyUpdateMsg constructApiKeyUpdatedMsg(UpdateMsgType msgType, ApiKey apiKey) { + return ApiKeyUpdateMsg.newBuilder().setMsgType(msgType).setEntity(JacksonUtil.toString(apiKey)) + .setIdMSB(apiKey.getId().getId().getMostSignificantBits()) + .setIdLSB(apiKey.getId().getId().getLeastSignificantBits()).build(); + } + + public static ApiKeyUpdateMsg constructApiKeyDeleteMsg(ApiKeyId apiKeyId) { + return ApiKeyUpdateMsg.newBuilder() + .setMsgType(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE) + .setIdMSB(apiKeyId.getId().getMostSignificantBits()) + .setIdLSB(apiKeyId.getId().getLeastSignificantBits()).build(); + } + public static List mergeAndFilterDownlinkDuplicates(List edgeEvents) { try { edgeEvents = removeDownlinkDuplicates(edgeEvents); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index c5822cc494..072a4878d5 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -19,7 +19,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; import io.grpc.stub.StreamObserver; import jakarta.annotation.Nullable; import jakarta.annotation.PreDestroy; @@ -37,7 +40,8 @@ import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.ResourceUtils; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.transport.config.ssl.PemSslCredentials; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; @@ -67,7 +71,6 @@ import org.thingsboard.server.service.edge.EdgeContextComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -112,6 +115,8 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private String certFileResource; @Value("${edges.rpc.ssl.private_key}") private String privateKeyResource; + @Value("${edges.rpc.ssl.key_password:}") + private String keyPassword; @Value("${edges.state.persistToTelemetry:false}") private boolean persistToTelemetry; @Value("${edges.rpc.client_max_keep_alive_time_sec:1}") @@ -173,9 +178,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i .addService(this); if (sslEnabled) { try { - InputStream certFileIs = ResourceUtils.getInputStream(this, certFileResource); - InputStream privateKeyFileIs = ResourceUtils.getInputStream(this, privateKeyResource); - builder.useTransportSecurity(certFileIs, privateKeyFileIs); + setupSsl(builder); } catch (Exception e) { log.error("Unable to set up SSL context. Reason: " + e.getMessage(), e); throw new RuntimeException("Unable to set up SSL context!", e); @@ -196,6 +199,33 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i log.info("Edge RPC service initialized!"); } + /** + * Configures TLS for the Edge gRPC server. + *

+ * Delegates PEM parsing and key management to {@link PemSslCredentials} — the same + * class used by MQTT, CoAP, and LwM2M transports — which supports: + *

+ * Path resolution (for both {@code cert} and {@code private_key}) is handled by + * {@link org.thingsboard.server.common.data.ResourceUtils#getInputStream ResourceUtils}: + * absolute path → relative / working-dir → classpath → {@code classpath:} prefix. + */ + void setupSsl(NettyServerBuilder builder) throws Exception { + PemSslCredentials credentials = new PemSslCredentials(); + credentials.setCertFile(certFileResource); + credentials.setKeyFile(StringUtils.isEmpty(privateKeyResource) ? null : privateKeyResource); + credentials.setKeyPassword(keyPassword); + credentials.init(false); + + SslContext sslContext = GrpcSslContexts.configure( + SslContextBuilder.forServer(credentials.createKeyManagerFactory())).build(); + builder.sslContext(sslContext); + } + @PreDestroy public void destroy() { if (server != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java index 474a6860d4..de4a92ae8c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSession.java @@ -49,6 +49,7 @@ import org.thingsboard.server.common.data.page.TimePageLink; import org.thingsboard.server.common.msg.edge.EdgeEventUpdateMsg; import org.thingsboard.server.dao.edge.stats.EdgeStatsKey; import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; @@ -988,6 +989,16 @@ public abstract class EdgeGrpcSession implements Closeable { } } } + if (uplinkMsg.getApiKeyUpdateMsgCount() > 0) { + for (ApiKeyUpdateMsg apiKeyUpdateMsg : uplinkMsg.getApiKeyUpdateMsgList()) { + sequenceDependencyLock.lock(); + try { + result.add(ctx.getApiKeyProcessor().processApiKeyMsgFromEdge(edge.getTenantId(), edge, apiKeyUpdateMsg)); + } finally { + sequenceDependencyLock.unlock(); + } + } + } } catch (Exception e) { String failureMsg = String.format("Can't process uplink msg [%s] from edge", uplinkMsg); log.trace("[{}][{}] Can't process uplink msg [{}]", tenantId, edge.getId(), uplinkMsg, e); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java index 207bbc2ce8..8455d86833 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/BaseEdgeProcessor.java @@ -146,7 +146,7 @@ public abstract class BaseEdgeProcessor implements EdgeProcessor { UPDATED_COMMENT, DELETED -> true; default -> switch (type) { case ALARM, ALARM_COMMENT, RULE_CHAIN, RULE_CHAIN_METADATA, USER, CUSTOMER, TENANT, TENANT_PROFILE, - WIDGETS_BUNDLE, WIDGET_TYPE, ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, CALCULATED_FIELD, AI_MODEL, NOTIFICATION_TEMPLATE, + WIDGETS_BUNDLE, WIDGET_TYPE, ADMIN_SETTINGS, OTA_PACKAGE, QUEUE, RELATION, CALCULATED_FIELD, AI_MODEL, API_KEY, NOTIFICATION_TEMPLATE, NOTIFICATION_TARGET, NOTIFICATION_RULE -> true; default -> false; }; diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyEdgeProcessor.java new file mode 100644 index 0000000000..960c0dd695 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyEdgeProcessor.java @@ -0,0 +1,105 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.apikey; + +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.EdgeUtils; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.edge.EdgeEvent; +import org.thingsboard.server.common.data.edge.EdgeEventType; +import org.thingsboard.server.common.data.id.ApiKeyId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.exception.DataValidationException; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.gen.edge.v1.DownlinkMsg; +import org.thingsboard.server.gen.edge.v1.EdgeVersion; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; + +import java.util.UUID; + +@Slf4j +@Component +@TbCoreComponent +public class ApiKeyEdgeProcessor extends BaseApiKeyProcessor implements ApiKeyProcessor { + + @Override + public ListenableFuture processApiKeyMsgFromEdge(TenantId tenantId, Edge edge, ApiKeyUpdateMsg apiKeyUpdateMsg) { + ApiKeyId apiKeyId = new ApiKeyId(new UUID(apiKeyUpdateMsg.getIdMSB(), apiKeyUpdateMsg.getIdLSB())); + try { + edgeSynchronizationManager.getEdgeId().set(edge.getId()); + + return switch (apiKeyUpdateMsg.getMsgType()) { + case ENTITY_CREATED_RPC_MESSAGE, ENTITY_UPDATED_RPC_MESSAGE -> { + boolean created = saveOrUpdateApiKey(tenantId, apiKeyId, apiKeyUpdateMsg); + if (created) { + ApiKey apiKey = edgeCtx.getApiKeyService().findApiKeyById(tenantId, apiKeyId); + if (apiKey != null) { + pushEntityEventToRuleEngine(tenantId, edge, apiKey, TbMsgType.ENTITY_CREATED); + } + } + yield Futures.immediateFuture(null); + } + case ENTITY_DELETED_RPC_MESSAGE -> { + deleteApiKey(tenantId, edge, apiKeyId); + yield Futures.immediateFuture(null); + } + default -> handleUnsupportedMsgType(apiKeyUpdateMsg.getMsgType()); + }; + } catch (DataValidationException e) { + return Futures.immediateFailedFuture(e); + } finally { + edgeSynchronizationManager.getEdgeId().remove(); + } + } + + @Override + public DownlinkMsg convertEdgeEventToDownlink(EdgeEvent edgeEvent, EdgeVersion edgeVersion) { + ApiKeyId apiKeyId = new ApiKeyId(edgeEvent.getEntityId()); + switch (edgeEvent.getAction()) { + case ADDED, UPDATED -> { + ApiKey apiKey = edgeCtx.getApiKeyService().findApiKeyById(edgeEvent.getTenantId(), apiKeyId); + if (apiKey != null) { + UpdateMsgType msgType = getUpdateMsgType(edgeEvent.getAction()); + ApiKeyUpdateMsg apiKeyUpdateMsg = EdgeMsgConstructorUtils.constructApiKeyUpdatedMsg(msgType, apiKey); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addApiKeyUpdateMsg(apiKeyUpdateMsg) + .build(); + } + } + case DELETED -> { + ApiKeyUpdateMsg apiKeyUpdateMsg = EdgeMsgConstructorUtils.constructApiKeyDeleteMsg(apiKeyId); + return DownlinkMsg.newBuilder() + .setDownlinkMsgId(EdgeUtils.nextPositiveInt()) + .addApiKeyUpdateMsg(apiKeyUpdateMsg) + .build(); + } + } + return null; + } + + @Override + public EdgeEventType getEdgeEventType() { + return EdgeEventType.API_KEY; + } +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyProcessor.java new file mode 100644 index 0000000000..f50594c9b7 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/ApiKeyProcessor.java @@ -0,0 +1,28 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.apikey; + +import com.google.common.util.concurrent.ListenableFuture; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.EdgeProcessor; + +public interface ApiKeyProcessor extends EdgeProcessor { + + ListenableFuture processApiKeyMsgFromEdge(TenantId tenantId, Edge edge, ApiKeyUpdateMsg apiKeyUpdateMsg); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java new file mode 100644 index 0000000000..7d110c3670 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/apikey/BaseApiKeyProcessor.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc.processor.apikey; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import lombok.extern.slf4j.Slf4j; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.id.ApiKeyId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.msg.TbMsgType; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.service.edge.rpc.processor.BaseEdgeProcessor; + +@Slf4j +public abstract class BaseApiKeyProcessor extends BaseEdgeProcessor { + + protected boolean saveOrUpdateApiKey(TenantId tenantId, ApiKeyId apiKeyId, ApiKeyUpdateMsg apiKeyUpdateMsg) { + boolean isCreated = false; + try { + ApiKey apiKey = JacksonUtil.fromString(apiKeyUpdateMsg.getEntity(), ApiKey.class, true); + if (apiKey == null) { + throw new RuntimeException("[{" + tenantId + "}] apiKeyUpdateMsg {" + apiKeyUpdateMsg + " } cannot be converted to apiKey"); + } + + ApiKey existingApiKey = edgeCtx.getApiKeyService().findApiKeyById(tenantId, apiKeyId); + if (existingApiKey == null) { + apiKey.setCreatedTime(Uuids.unixTimestamp(apiKeyId.getId())); + isCreated = true; + } + + apiKey.setId(apiKeyId); + edgeCtx.getApiKeyService().saveApiKey(tenantId, apiKey, apiKey.getValue(), false); + } catch (Exception e) { + log.error("[{}] Failed to process apiKey update msg [{}]", tenantId, apiKeyUpdateMsg, e); + throw e; + } + return isCreated; + } + + protected void deleteApiKey(TenantId tenantId, Edge edge, ApiKeyId apiKeyId) { + ApiKey apiKey = edgeCtx.getApiKeyService().findApiKeyById(tenantId, apiKeyId); + if (apiKey != null) { + edgeCtx.getApiKeyService().deleteApiKey(tenantId, apiKey, false); + pushEntityEventToRuleEngine(tenantId, edge, apiKey, TbMsgType.ENTITY_DELETED); + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java index 027b1225c4..03fc29ae3c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/processor/user/UserEdgeProcessor.java @@ -23,6 +23,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EdgeUtils; import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.edge.EdgeEventActionType; @@ -34,6 +35,7 @@ import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.msg.TbMsgMetaData; import org.thingsboard.server.exception.DataValidationException; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.DownlinkMsg; import org.thingsboard.server.gen.edge.v1.EdgeVersion; import org.thingsboard.server.gen.edge.v1.UpdateMsgType; @@ -42,6 +44,7 @@ import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.edge.EdgeMsgConstructorUtils; +import java.util.List; import java.util.UUID; @Slf4j @@ -135,6 +138,10 @@ public class UserEdgeProcessor extends BaseUserProcessor implements UserProcesso if (userCredentialsByUserId != null) { builder.addUserCredentialsUpdateMsg(EdgeMsgConstructorUtils.constructUserCredentialsUpdatedMsg(userCredentialsByUserId)); } + List apiKeys = edgeCtx.getApiKeyService().findApiKeysByUserId(edgeEvent.getTenantId(), userId); + for (ApiKey apiKey : apiKeys) { + builder.addApiKeyUpdateMsg(EdgeMsgConstructorUtils.constructApiKeyUpdatedMsg(msgType, apiKey)); + } return builder.build(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java b/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java index 162a780bbb..46af0120da 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/pat/ApiKeyAuthenticationProvider.java @@ -46,7 +46,7 @@ public class ApiKeyAuthenticationProvider extends AbstractAuthenticationProvider return new ApiKeyAuthenticationToken(securityUser); } - private SecurityUser authenticate(String key) { + public SecurityUser authenticate(String key) { if (StringUtils.isEmpty(key)) { throw new BadCredentialsException("Empty API key"); } diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index f0f2e1eaa7..fc8c5c3be2 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -34,6 +34,7 @@ import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.dao.nosql.ResultSetSizeLimitExceededException; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -242,7 +243,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc @Override public void onFailure(Throwable t) { - log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId()); + log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId(), t); + if (t instanceof ResultSetSizeLimitExceededException) { + sendError(finalCtx, t); + } } }, wsCallBackExecutor); } @@ -258,7 +262,18 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc handleLatestCmd(ctx, cmd.getLatestCmd()); } if (cmd.getTsCmd() != null) { - handleTimeSeriesCmd(ctx, cmd.getTsCmd()); + Futures.addCallback(handleTimeSeriesCmd(ctx, cmd.getTsCmd()), new FutureCallback<>() { + @Override + public void onSuccess(TbEntityDataSubCtx result) {} + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}] Failed to process timeseries command", ctx.getSessionId(), ctx.getCmdId(), t); + if (t instanceof ResultSetSizeLimitExceededException) { + sendError(ctx, t); + } + } + }, wsCallBackExecutor); } } else { checkAndSendInitialData(ctx); @@ -268,6 +283,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } } + private void sendError(TbEntityDataSubCtx ctx, Throwable t) { + ctx.sendWsMsg(new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), t.getMessage())); + } + private void checkAndSendInitialData(@Nullable TbEntityDataSubCtx theCtx) { if (!theCtx.isInitialDataSent()) { EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription()); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 51663eea9c..2b1a810924 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -348,7 +348,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer if (sub.isLatestValues()) { for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); - if (stateTs == null || kv.getTs() >= stateTs) { + if (stateTs == null || kv.getTs() >= stateTs || kv.isDeletedEntry()) { if (updateData == null) { updateData = new ArrayList<>(); } @@ -362,7 +362,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); if (stateTs != null) { - if (!sub.isLatestValues() || kv.getTs() >= stateTs) { + if (!sub.isLatestValues() || kv.getTs() >= stateTs || kv.isDeletedEntry()) { if (updateData == null) { updateData = new ArrayList<>(); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 5865d9e39d..596f5a8754 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -32,4 +32,8 @@ public abstract class AbstractCleanUpService { return partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition(); } + protected boolean isTenantPartitionMine(TenantId tenantId) { + return partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java index ddc95b74e9..83855bc8f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java @@ -20,33 +20,41 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.NotificationRequestConfig; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.notification.NotificationRequestDao; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; +import java.time.Instant; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.dao.model.ModelConstants.NOTIFICATION_TABLE_NAME; +@Slf4j @Service @ConditionalOnExpression("${sql.ttl.notifications.enabled:true} && ${sql.ttl.notifications.ttl:0} > 0") -@Slf4j public class NotificationsCleanUpService extends AbstractCleanUpService { private final SqlPartitioningRepository partitioningRepository; private final NotificationRequestDao notificationRequestDao; + private final TenantService tenantService; @Value("${sql.ttl.notifications.ttl:2592000}") private long ttlInSec; @Value("${sql.notifications.partition_size:168}") private int partitionSizeInHours; + @Value("${sql.ttl.notifications.removal_batch_size:10000}") + private int removalBatchSize; public NotificationsCleanUpService(PartitionService partitionService, SqlPartitioningRepository partitioningRepository, - NotificationRequestDao notificationRequestDao) { + NotificationRequestDao notificationRequestDao, TenantService tenantService) { super(partitionService); this.partitioningRepository = partitioningRepository; this.notificationRequestDao = notificationRequestDao; + this.tenantService = tenantService; } @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.notifications.checking_interval_ms:86400000})}", @@ -54,18 +62,65 @@ public class NotificationsCleanUpService extends AbstractCleanUpService { public void cleanUp() { long expTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttlInSec); long partitionDurationMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); - if (!isSystemTenantPartitionMine()) { + if (isSystemTenantPartitionMine()) { + partitioningRepository.dropPartitionsBefore(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); + } else { partitioningRepository.cleanupPartitionsCache(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); - return; } - long lastRemovedNotificationTs = partitioningRepository.dropPartitionsBefore(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); - if (lastRemovedNotificationTs > 0) { - long gap = TimeUnit.MINUTES.toMillis(10); - long requestExpTime = lastRemovedNotificationTs - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; - int removed = notificationRequestDao.removeAllByCreatedTimeBefore(requestExpTime); - log.info("Removed {} outdated notification requests older than {}", removed, requestExpTime); + long gap = TimeUnit.MINUTES.toMillis(10); + long requestExpTime = expTime - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; + cleanUpNotificationRequests(requestExpTime); + } + + private void cleanUpNotificationRequests(long expirationTime) { + log.info("Starting notification requests cleanup for records older than {}", Instant.ofEpochMilli(expirationTime)); + int totalRemoved = 0; + int tenantsProcessed = 0; + + // Clean up SYSADMIN's notification requests on the system node only + if (isSystemTenantPartitionMine()) { + try { + totalRemoved += cleanUpByTenant(TenantId.SYS_TENANT_ID, expirationTime); + } catch (Exception e) { + log.warn("Failed to clean up notification requests for sysadmin {}", TenantId.SYS_TENANT_ID, e); + } + } + // Each node cleans up notification requests for its own tenants + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + if (!isTenantPartitionMine(tenantId)) { + continue; + } + int tenantRemoved = cleanUpByTenant(tenantId, expirationTime); + totalRemoved += tenantRemoved; + tenantsProcessed++; + if (tenantRemoved > 0) { + log.trace("Removed {} notification requests for tenant {}", tenantRemoved, tenantId); + } + } catch (Exception e) { + log.warn("Failed to clean up notification requests for tenant {}", tenantId, e); + } } + + log.info("Notification requests cleanup completed. Processed {} tenants, removed {} total records older than {}", tenantsProcessed, totalRemoved, Instant.ofEpochMilli(expirationTime)); + } + + private int cleanUpByTenant(TenantId tenantId, long expirationTime) { + int totalRemoved = 0; + int batchRemoved; + + do { + batchRemoved = notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, removalBatchSize); + totalRemoved += batchRemoved; + + if (batchRemoved > 0) { + log.trace("Removed {} notification requests in batch for tenant {}", batchRemoved, tenantId); + } + } while (batchRemoved >= removalBatchSize); + + return totalRemoved; } } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java index 9404a0ae72..a945bfd8a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java @@ -15,69 +15,87 @@ */ package org.thingsboard.server.service.ttl.rpc; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; -import java.util.Date; +import java.time.Instant; import java.util.Optional; import java.util.concurrent.TimeUnit; -@TbCoreComponent -@Service @Slf4j -@RequiredArgsConstructor -public class RpcCleanUpService { - @Value("${sql.ttl.rpc.enabled}") - private boolean ttlTaskExecutionEnabled; +@Service +@TbCoreComponent +@ConditionalOnExpression("${sql.ttl.rpc.enabled:true}") +public class RpcCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.rpc.removal_batch_size:10000}") + private int removalBatchSize; + private final RpcDao rpcDao; private final TenantService tenantService; - private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; - private final RpcDao rpcDao; + + public RpcCleanUpService(TenantService tenantService, PartitionService partitionService, TbTenantProfileCache tenantProfileCache, RpcDao rpcDao) { + super(partitionService); + this.tenantService = tenantService; + this.tenantProfileCache = tenantProfileCache; + this.rpcDao = rpcDao; + } @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.rpc.checking_interval})}", fixedDelayString = "${sql.ttl.rpc.checking_interval}") public void cleanUp() { - if (ttlTaskExecutionEnabled) { - PageLink tenantsBatchRequest = new PageLink(10_000, 0); - PageData tenantsIds; - do { - tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); - for (TenantId tenantId : tenantsIds.getData()) { - if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { - continue; - } - - Optional tenantProfileConfiguration = tenantProfileCache.get(tenantId).getProfileConfiguration(); - if (tenantProfileConfiguration.isEmpty() || tenantProfileConfiguration.get().getRpcTtlDays() == 0) { - continue; - } - - long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays()); - long expirationTime = System.currentTimeMillis() - ttl; - - int totalRemoved = rpcDao.deleteOutdatedRpcByTenantId(tenantId, expirationTime); - - if (totalRemoved > 0) { - log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, new Date(expirationTime)); - } + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + if (!isTenantPartitionMine(tenantId)) { + continue; } - tenantsBatchRequest = tenantsBatchRequest.nextPageLink(); - } while (tenantsIds.hasNext()); + Optional tenantProfileConfiguration = tenantProfileCache.get(tenantId).getProfileConfiguration(); + if (tenantProfileConfiguration.isEmpty() || tenantProfileConfiguration.get().getRpcTtlDays() == 0) { + continue; + } + + long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays()); + long expirationTime = System.currentTimeMillis() - ttl; + + int totalRemoved = cleanUpByTenant(tenantId, expirationTime); + + if (totalRemoved > 0) { + log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, Instant.ofEpochMilli(expirationTime)); + } + } catch (Exception e) { + log.warn("Failed to clean up rpc by ttl for tenant {}", tenantId, e); + } } } + private int cleanUpByTenant(TenantId tenantId, long expirationTime) { + int totalRemoved = 0; + int batchRemoved; + + do { + batchRemoved = rpcDao.deleteOutdatedRpcByTenantIdBatch(tenantId, expirationTime, removalBatchSize); + totalRemoved += batchRemoved; + + if (batchRemoved > 0) { + log.trace("Removed {} rpc in batch for tenant {}", batchRemoved, tenantId); + } + } while (batchRemoved >= removalBatchSize); + + return totalRemoved; + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java b/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java index e139498d04..a2dc87e4af 100644 --- a/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java +++ b/application/src/main/java/org/thingsboard/server/service/ws/AuthCmd.java @@ -23,11 +23,14 @@ import lombok.NoArgsConstructor; @NoArgsConstructor @AllArgsConstructor public class AuthCmd implements WsCmd { + private int cmdId; private String token; + private String apiKey; @Override public WsCmdType getType() { return WsCmdType.AUTH; } + } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2131407cd2..da73af2b55 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -15,6 +15,7 @@ # # Server common parameters +# Configures HTTP/HTTPS bind address, port, SSL, WebSocket, and REST API settings. server: # Server bind-address address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -109,23 +110,25 @@ server: response_timeout: "${DEFAULT_RULE_ENGINE_RESPONSE_TIMEOUT:10000}" # Application info parameters +# Exposes application metadata such as version string injected at build time. app: # Application version version: "@project.version@" -# Zookeeper connection parameters +# ZooKeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. @@ -133,6 +136,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cluster parameters +# Controls cluster statistics — tracks the number of messages exchanged between cluster nodes. cluster: stats: # Enable/Disable the cluster statistics. Calculates the number of messages sent between cluster nodes based on each type @@ -141,11 +145,13 @@ cluster: print_interval_ms: "${TB_CLUSTER_STATS_PRINT_INTERVAL_MS:10000}" # Plugins configuration parameters +# Defines classpath scan packages used to discover and register ThingsBoard extension plugins. plugins: # Comma-separated package list used during classpath scanning for plugins scan_packages: "${PLUGINS_SCAN_PACKAGES:org.thingsboard.server.extensions,org.thingsboard.rule.engine}" # Security parameters +# Configures JWT tokens, user login, device claiming, OAuth2, and CA certificate trust store. security: # JWT Token parameters jwt: # Since 3.4.2 values are persisted in the database during installation or upgrade. On Install, the key will be generated randomly if no custom value set. You can change it later from Web UI under SYS_ADMIN @@ -184,6 +190,7 @@ security: password: "${SECURITY_JAVA_CACERTS_PASSWORD:changeit}" # Mail settings parameters +# Configures mail service OAuth2 token refresh and per-tenant sending rate limits. mail: oauth2: # Interval for checking refresh token expiration in seconds(by default, 1 day). @@ -192,6 +199,7 @@ mail: per_tenant_rate_limits: "${MAIL_PER_TENANT_RATE_LIMITS:}" # Usage statistics parameters +# Controls collection and reporting intervals for API usage stats at system, tenant, and customer levels. usage: stats: report: @@ -215,6 +223,7 @@ usage: report_interval: "${DEVICES_STATS_REPORT_INTERVAL:60}" # UI settings parameters +# Configures dashboard data limits and base URL for UI help assets. ui: # Dashboard parameters dashboard: @@ -226,6 +235,7 @@ ui: base-url: "${UI_HELP_BASE_URL:https://raw.githubusercontent.com/thingsboard/thingsboard-ui-help/release-4.4}" # Database telemetry parameters +# Selects the storage backend (SQL, Cassandra, or TimescaleDB) for time-series and latest telemetry data. database: ts_max_intervals: "${DATABASE_TS_MAX_INTERVALS:700}" # Max number of DB queries generated by a single API call to fetch telemetry records ts: @@ -234,6 +244,7 @@ database: type: "${DATABASE_TS_LATEST_TYPE:sql}" # cassandra, sql, or timescale (for hybrid mode, DATABASE_TS_TYPE value should be cassandra, or timescale) # Cassandra driver configuration parameters +# Configures cluster name, keyspace, SSL, credentials, socket, and query settings for the Cassandra driver. cassandra: # Thingsboard cluster name cluster_name: "${CASSANDRA_CLUSTER_NAME:Thingsboard Cluster}" @@ -352,6 +363,7 @@ cassandra: print_tenant_names: "${CASSANDRA_QUERY_TENANT_RATE_LIMITS_PRINT_TENANT_NAMES:false}" # SQL configuration parameters +# Tunes batch sizes, delays, thread counts, TTL, and partitioning for SQL-backed persistence of telemetry, events, and audit logs. sql: # Specify batch size for persisting attribute updates attributes: @@ -432,10 +444,11 @@ sql: edge_events_ttl: "${SQL_TTL_EDGE_EVENTS_TTL:2628000}" # Number of seconds. The current value corresponds to one month alarms: checking_interval: "${SQL_ALARMS_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours - removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # To delete outdated alarms not all at once but in batches + removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # Batch size for records removal rpc: enabled: "${SQL_TTL_RPC_ENABLED:true}" # Enable/disable TTL (Time To Live) for rpc call records checking_interval: "${SQL_RPC_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours + removal_batch_size: "${SQL_RPC_TTL_REMOVAL_BATCH_SIZE:10000}" # Batch size for records removal audit_logs: enabled: "${SQL_TTL_AUDIT_LOGS_ENABLED:true}" # Enable/disable TTL (Time To Live) for audit log records ttl: "${SQL_TTL_AUDIT_LOGS_SECS:0}" # Disabled by default. The accuracy of the cleanup depends on the sql.audit_logs.partition_size @@ -444,6 +457,7 @@ sql: enabled: "${SQL_TTL_NOTIFICATIONS_ENABLED:true}" # Enable/disable TTL (Time To Live) for notification center records ttl: "${SQL_TTL_NOTIFICATIONS_SECS:2592000}" # Default value - 30 days checking_interval_ms: "${SQL_TTL_NOTIFICATIONS_CHECKING_INTERVAL_MS:86400000}" # Default value - 1 day + removal_batch_size: "${SQL_TTL_NOTIFICATIONS_REMOVAL_BATCH_SIZE:10000}" # Batch size for records removal api_keys: enabled: "${SQL_TTL_API_KEYS_ENABLED:true}" # Enable/disable TTL (Time To Live) for expired api keys records checking_interval_ms: "${SQL_TTL_API_KEYS_CHECKING_INTERVAL_MS:86400000}" # Default value - 1 day @@ -453,6 +467,7 @@ sql: query_timeout: "${SQL_RELATIONS_QUERY_TIMEOUT_SEC:20}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls # Actor system parameters +# Configures thread pools, timeouts, and behavior for the internal actor system processing devices, rules, and RPCs. actors: system: throughput: "${ACTORS_SYSTEM_THROUGHPUT:5}" # Number of messages the actor system will process per actor before switching to processing of messages for the next actor @@ -553,6 +568,8 @@ actors: # Time in seconds to receive calculation result. calculation_timeout: "${ACTORS_CALCULATION_TIMEOUT_SEC:5}" +# Debug settings parameters +# Configures the global default duration for debug mode used by rule chains and calculated fields. debug: settings: # Default duration (in minutes) for debug mode. Min value is 1 minute. Tenant profile settings override this one. @@ -560,6 +577,7 @@ debug: default_duration: "${DEBUG_SETTINGS_DEFAULT_DURATION_MINUTES:15}" # Cache settings parameters +# Configures cache type (Caffeine or Redis), pool size, and per-entity TTL and max-size limits for all caches. cache: # caffeine or redis(7.2 - latest compatible version) type: "${CACHE_TYPE:caffeine}" @@ -719,6 +737,7 @@ cache: spring.data.redis.repositories.enabled: false # Disable this because it is not required. # Redis/Valkey configuration parameters +# Configures standalone, cluster, or sentinel Redis connection, SSL, and connection pool settings used for caching. redis: # standalone or cluster or sentinel connection: @@ -755,9 +774,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # If set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -802,11 +821,13 @@ redis: # Update version parameters +# Controls whether the platform periodically checks for new ThingsBoard releases. updates: # Enable/disable checks for the new version enabled: "${UPDATES_ENABLED:true}" # Spring CORS configuration parameters +# Defines allowed origins, methods, headers, and credentials for cross-origin REST API requests. spring.mvc.cors: mappings: # Intercept path @@ -823,6 +844,7 @@ spring.mvc.cors: allow-credentials: "true" # General spring parameters +# Miscellaneous Spring Boot settings for circular references, Freemarker, MVC, multipart, and JPA dialect. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. spring.freemarker.checkTemplateLocation: "false" # spring freemarker configuration spring.mvc.async.request-timeout: "${SPRING_MVC_ASYNC_REQUEST_TIMEOUT:30000}" # The default timeout for asynchronous requests in milliseconds @@ -841,6 +863,7 @@ spring.jpa.properties.hibernate.order_by.default_null_ordering: "${SPRING_JPA_PR spring.jpa.properties.hibernate.dialect: "${SPRING_JPA_DIALECT:org.thingsboard.server.dao.ThingsboardPostgreSQLDialect}" # we use custom dialect that contains ilike(arg1, arg2) function (is interpreted to postgres ILIKE operator) # SQL DAO Configuration parameters +# Configures Spring JPA, Hibernate DDL, and HikariCP datasource connections for the primary and events databases. spring: data: jpa: @@ -891,6 +914,7 @@ spring: registerMbeans: "${SPRING_EVENTS_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # Audit log parameters +# Configures audit logging levels per entity type and optional forwarding to an external sink such as Elasticsearch. audit-log: # Enable/disable audit log functionality. enabled: "${AUDIT_LOG_ENABLED:true}" @@ -933,6 +957,7 @@ audit-log: password: "${AUDIT_LOG_SINK_PASSWORD:}" # Password used to access external sink system # Device state parameters +# Configures device inactivity detection, state persistence strategy, and rate limits for state rule nodes. state: # Device inactivity timeout is a global configuration parameter that defines when the device will be marked as "inactive" by the server. # The parameter value is in seconds. A user can overwrite this parameter for an individual device by setting the “inactivityTimeout” server-side attribute (NOTE: expects value in milliseconds). @@ -963,7 +988,8 @@ state: # Refill is set to be greedy. Please refer to Bucket4j library documentation for more details. rateLimit: "${DEVICE_STATE_NODE_RATE_LIMIT_CONFIGURATION:1:1,30:60,60:3600}" -# Tbel parameters +# TBEL parameters +# Configures the ThingsBoard Expression Language (TBEL) engine: limits, timeouts, thread pool, and blacklisting. tbel: # Enable/Disable TBEL feature. enabled: "${TBEL_ENABLED:true}" @@ -992,6 +1018,7 @@ tbel: print_interval_ms: "${TB_TBEL_STATS_PRINT_INTERVAL_MS:10000}" # JS parameters +# Configures the JavaScript execution engine (local Nashorn or remote Node.js): limits, sandboxing, and thread pools. js: # local (Nashorn Engine, deprecated) OR remote JS-Executors (NodeJS) evaluator: "${JS_EVALUATOR:local}" @@ -1039,6 +1066,7 @@ js: print_interval_ms: "${TB_JS_REMOTE_STATS_PRINT_INTERVAL_MS:10000}" # Transport configuration parameters +# Configures session management, rate limits, and protocol-specific settings for HTTP, MQTT, CoAP, LwM2M, and SNMP transports. transport: sessions: # Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device. @@ -1349,6 +1377,7 @@ transport: fetch_frequency: "${TB_GATEWAY_DASHBOARD_SYNC_FETCH_FREQUENCY:24}" # CoAP server parameters +# Configures the standalone CoAP server bind address, port, DTLS encryption, and credential settings. coap: server: # Enable/disable coap server. @@ -1433,6 +1462,7 @@ coap: dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" # Device connectivity parameters +# Specifies the hosts, ports, and credentials exposed to the UI for generating device connection check commands. device: connectivity: http: @@ -1486,6 +1516,7 @@ device: image_version: "${DEVICE_CONNECTIVITY_GATEWAY_IMAGE_VERSION:3.8-stable}" # Edges parameters +# Controls Edge instance gRPC communication, event storage, state persistence, and statistics reporting. edges: # Enable/disable Edge instance enabled: "${EDGES_ENABLED:true}" @@ -1507,10 +1538,17 @@ edges: ssl: # Enable/disable SSL support enabled: "${EDGES_RPC_SSL_ENABLED:false}" - # Cert file to be used during TLS connectivity to the cloud + # Path to the server certificate file (holds server certificate or certificate chain, may include server private key). + # Accepts an absolute filesystem path (e.g. /etc/thingsboard/certChainFile.pem), + # a relative path resolved against the working directory first then the classpath, + # or a classpath resource with the explicit "classpath:" prefix (e.g. classpath:conf/certChainFile.pem). cert: "${EDGES_RPC_SSL_CERT:certChainFile.pem}" - # Private key file associated with the Cert certificate. This key is used in the encryption process during a secure connection + # Path to the server certificate private key file. Optional if the private key is already present in the cert file above. + # Supports the same path resolution as 'cert': absolute, relative/classpath, or "classpath:" prefix. + # Leave empty when using a combined PEM cert that already contains the private key. private_key: "${EDGES_RPC_SSL_PRIVATE_KEY:privateKeyFile.pem}" + # Server certificate private key password (optional). Leave empty if the key is not encrypted. + key_password: "${EDGES_RPC_SSL_KEY_PASSWORD:}" # Maximum size (in bytes) of inbound messages the cloud can handle from the edge. By default, it can handle messages up to 4 Megabytes max_inbound_message_size: "${EDGES_RPC_MAX_INBOUND_MESSAGE_SIZE:4194304}" # Maximum length of telemetry (time-series and attributes) message the cloud sends to the edge. By default, there is no limitation. @@ -1547,6 +1585,7 @@ edges: report-interval-millis: "${EDGES_STATS_REPORT_INTERVAL_MS:600000}" # Spring doc common parameters +# Enables or disables the OpenAPI/Swagger documentation endpoint and sets the default media type. springdoc: # If false swagger API docs will be unavailable api-docs.enabled: "${SWAGGER_ENABLED:true}" @@ -1554,6 +1593,7 @@ springdoc: default-produces-media-type: "${SWAGGER_DEFAULT_PRODUCES_MEDIA_TYPE:application/json}" # Swagger common parameters +# Configures Swagger UI metadata: title, description, contact, license, version, and API path patterns. swagger: # General swagger match pattern of swagger UI links api_path: "${SWAGGER_API_PATH:/api/**}" @@ -1585,6 +1625,7 @@ swagger: doc_expansion: "${SWAGGER_DOC_EXPANSION:list}" # Queue configuration parameters +# Configures the message queue backend (in-memory or Kafka) and all topic/partition/consumer settings for each service. queue: type: "${TB_QUEUE_TYPE:in-memory}" # in-memory or kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -1998,12 +2039,14 @@ queue: processing_interval: "${TB_QUEUE_TASKS_STATS_PROCESSING_INTERVAL_MS:1000}" # Event configuration parameters +# Controls limits on debug event content size stored for rule chain and rule node execution. event: debug: # Maximum number of symbols per debug event. The event content will be truncated if needed max-symbols: "${TB_MAX_DEBUG_EVENT_SYMBOLS:4096}" # General service parameters +# Sets the deployment type (monolith or microservice) and assigns tenant profiles to specific Rule Engine instances. service: type: "${TB_SERVICE_TYPE:monolith}" # monolith or tb-core or tb-rule-engine # Unique id for this service (autogenerated if empty) @@ -2017,6 +2060,7 @@ service: executor_thread_pool_size: "${TB_RULE_ENGINE_PUBSUB_EXECUTOR_THREAD_POOL_SIZE:0}" # Metrics parameters +# Enables actuator metrics endpoint and configures system info (CPU, memory) persistence frequency and TTL. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" @@ -2030,6 +2074,7 @@ metrics: ttl: "${METRICS_SYSTEM_INFO_TTL_DAYS:7}" # Version control parameters +# Configures thread pools and Git repository folder for entity version control (export/import) operations. vc: # Pool size for handling export tasks thread_pool_size: "${TB_VC_POOL_SIZE:6}" @@ -2040,6 +2085,7 @@ vc: repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" # Notification system parameters +# Configures thread pool size and deduplication intervals for the notification rule processing engine. notification_system: # Specify thread pool size for Notification System processing notification rules and notification sending. Recommend value <= 10 thread_pool_size: "${TB_NOTIFICATION_SYSTEM_THREAD_POOL_SIZE:10}" @@ -2048,6 +2094,7 @@ notification_system: deduplication_durations: "${TB_NOTIFICATION_RULES_DEDUPLICATION_DURATIONS:NEW_PLATFORM_VERSION:0;RATE_LIMITS:14400000;}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure and health indicator settings. management: endpoints: web: @@ -2060,6 +2107,7 @@ management: enabled: "false" # Mobile application settings for Thingsboard mobile application +# Specifies domain name and store links used for the ThingsBoard Live mobile application QR code and deep links. mobileApp: # Server domain name for Thingsboard Live mobile application domain: "${TB_MOBILE_APP_DOMAIN:demo.thingsboard.io}" @@ -2068,6 +2116,8 @@ mobileApp: # Link to App Store for Thingsboard Live mobile application appStoreLink: "${TB_MOBILE_APP_APP_STORE_LINK:https://apps.apple.com/us/app/thingsboard-live/id1594355695}" +# MQTT client parameters +# Configures MQTT client retransmission behavior including max attempts, initial delay, and jitter factor. mqtt: # MQTT client configuration parameters client: diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java index c25dafa5e6..996efb17be 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractControllerTest.java @@ -86,12 +86,12 @@ public abstract class AbstractControllerTest extends AbstractNotifyEntityTest { } @Before - public void beforeWsTest() throws Exception { + public void beforeWsTest() { // placeholder } @After - public void afterWsTest() throws Exception { + public void afterWsTest() { if (wsClient != null) { wsClient.close(); } @@ -113,4 +113,11 @@ public abstract class AbstractControllerTest extends AbstractNotifyEntityTest { return wsClient; } + protected TbTestWebSocketClient buildAndConnectWebSocketClientWithApiKey(String apiKey) throws URISyntaxException, InterruptedException { + TbTestWebSocketClient wsClient = new TbTestWebSocketClient(new URI(WS_URL + wsPort + "/api/ws")); + assertThat(wsClient.connectBlocking(TIMEOUT, TimeUnit.SECONDS)).isTrue(); + wsClient.authenticateWithApiKey(apiKey); + return wsClient; + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index e8fcd6a723..0ff227384e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -425,6 +425,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { public void teardownWebTest() throws Exception { log.debug("Executing web test teardown"); + // Drain any pending housekeeper work left by the test body (e.g., bulk tenant deletes) + // before proceeding with teardown deletions, to avoid 90s per-tenant wait timing out. + awaitHousekeeperDrained(); + loginSysAdmin(); deleteTenant(tenantId); deleteDifferentTenant(); @@ -457,6 +461,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { .until(() -> storage.getLag("tb_housekeeper") == 0); } + protected void awaitHousekeeperDrained() { + Awaitility.await("housekeeper drained").atMost(5, TimeUnit.MINUTES).during(300, TimeUnit.MILLISECONDS) + .until(() -> storage.getLag("tb_housekeeper") == 0); + } + private List getAllTenants() throws Exception { List loadedTenants = new ArrayList<>(); PageLink pageLink = new PageLink(10); diff --git a/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java new file mode 100644 index 0000000000..405eecec9b --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/controller/ApiKeyWebSocketApiTest.java @@ -0,0 +1,71 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.controller; + +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.common.data.pat.ApiKeyInfo; +import org.thingsboard.server.dao.service.DaoSqlTest; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@Slf4j +@DaoSqlTest +public class ApiKeyWebSocketApiTest extends WebSocketApiTest { + + private ApiKey apiKey; + + @Before + public void setUpApiKey() { + ApiKeyInfo apiKeyInfo = new ApiKeyInfo(); + apiKeyInfo.setDescription("WS test API key"); + apiKeyInfo.setEnabled(true); + apiKeyInfo.setUserId(tenantAdminUserId); + apiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class); + } + + @After + public void tearDownApiKey() throws Exception { + loginTenantAdmin(); + doDelete("/api/apiKey/" + apiKey.getId()).andExpect(status().isOk()); + } + + @Override + protected TbTestWebSocketClient buildAndConnectWebSocketClient() throws URISyntaxException, InterruptedException { + return buildAndConnectWebSocketClientWithApiKey(apiKey.getValue()); + } + + @Test + public void testInvalidApiKeyAuthCmd_connectionClosed() throws Exception { + TbTestWebSocketClient client = new TbTestWebSocketClient(new URI(WS_URL + wsPort + "/api/ws")); + assertThat(client.connectBlocking(TIMEOUT, TimeUnit.SECONDS)).isTrue(); + try { + client.authenticateWithApiKey("invalid-key"); + assertThat(client.waitForClose()).isTrue(); + } finally { + client.close(); + } + } + +} diff --git a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java index af5df3d733..95582cf261 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AuditLogControllerTest.java @@ -19,6 +19,7 @@ import com.datastax.oss.driver.api.core.uuid.Uuids; import com.fasterxml.jackson.core.type.TypeReference; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -61,6 +62,9 @@ import static org.mockito.Mockito.verify; @DaoSqlTest public class AuditLogControllerTest extends AbstractControllerTest { + // Small enough to force multiple pages, verifying pagination loop correctness + private static final int SMALL_PAGE_SIZE = 5; + private Tenant savedTenant; private User tenantAdmin; @@ -112,17 +116,14 @@ public class AuditLogControllerTest extends AbstractControllerTest { doPost("/api/device", device, Device.class); } - List loadedAuditLogs = getAuditLogs(5, "/api/audit/logs?"); - - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); - - loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?"); - - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs?")).hasSize(11 + 1)); - loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?"); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/customer/" + ModelConstants.NULL_UUID + "?")).hasSize(11 + 1)); - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/user/" + tenantAdmin.getId().getId().toString() + "?")).hasSize(11 + 1)); } @Test @@ -136,9 +137,10 @@ public class AuditLogControllerTest extends AbstractControllerTest { doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); } - List loadedAuditLogs = getAuditLogs(100, "/api/audit/logs?"); - - Assert.assertEquals("Have X audit log before this test + New tenant profiles in the test", loadedAuditLogsBefore.size() + 3, loadedAuditLogs.size()); + int expectedSize = loadedAuditLogsBefore.size() + 3; + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + Assert.assertEquals("Have X audit log before this test + New tenant profiles in the test", + expectedSize, getAuditLogs(100, "/api/audit/logs?").size())); } private List getAuditLogs(int pageSize, String urlTemplate) throws Exception { @@ -168,9 +170,9 @@ public class AuditLogControllerTest extends AbstractControllerTest { savedDevice = doPost("/api/device", savedDevice, Device.class); } - List loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/entity/DEVICE/" + savedDevice.getId().getId() + "?"); - - Assert.assertEquals(11 + 1, loadedAuditLogs.size()); + Device finalSavedDevice = savedDevice; + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/DEVICE/" + finalSavedDevice.getId().getId() + "?")).hasSize(11 + 1)); } @Test @@ -186,9 +188,10 @@ public class AuditLogControllerTest extends AbstractControllerTest { tenantProfile.setName(tenantProfile.getName() + "(old)"); tenantProfile = doPost("/api/tenantProfile", tenantProfile, TenantProfile.class); - List loadedAuditLogs = getAuditLogs(5, "/api/audit/logs/entity/" +tenantProfile.getId().getEntityType()+ "/" + tenantProfile.getId().getId() + "?"); - - Assert.assertEquals("Audit logs count by Tenant Profile entity", 2, loadedAuditLogs.size()); + TenantProfile finalTenantProfile = tenantProfile; + Awaitility.await().atMost(TIMEOUT, TimeUnit.SECONDS).untilAsserted(() -> + assertThat(getAuditLogs(SMALL_PAGE_SIZE, "/api/audit/logs/entity/" + finalTenantProfile.getId().getEntityType() + "/" + finalTenantProfile.getId().getId() + "?")) + .as("Audit logs count by Tenant Profile entity").hasSize(2)); //cleanup doDelete("/api/tenantProfile/" + tenantProfile.getId().getId().toString()); diff --git a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java index d9ec187e39..18e11faaf9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/CalculatedFieldControllerTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.controller; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.EntityType; @@ -108,6 +109,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); assertThat(savedCalculatedField.getConfiguration()).isEqualTo(getSimpleCalculatedFieldConfig()); + assertThat(savedCalculatedField.getAdditionalInfo()).isEqualTo(calculatedField.getAdditionalInfo()); assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); savedCalculatedField.setName("Test CF"); @@ -115,6 +117,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { CalculatedField updatedCalculatedField = doPost("/api/calculatedField", savedCalculatedField, CalculatedField.class); assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getAdditionalInfo()).isEqualTo(savedCalculatedField.getAdditionalInfo()); assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); doDelete("/api/calculatedField/" + savedCalculatedField.getId().getId().toString()) @@ -322,6 +325,7 @@ public class CalculatedFieldControllerTest extends AbstractControllerTest { calculatedField.setType(cfType); calculatedField.setName("Test Calculated Field for " + entityId); calculatedField.setConfigurationVersion(1); + calculatedField.setAdditionalInfo(JacksonUtil.newObjectNode()); if (customConfiguration != null) { calculatedField.setConfiguration(customConfiguration); } else switch (cfType) { diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java index a4d6b62a0c..277ec687f1 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java @@ -41,6 +41,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.springframework.test.context.TestPropertySource; +import org.springframework.test.util.TestSocketUtils; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; @@ -90,8 +91,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; -import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest.MQTT_PORT; -import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest.MQTT_URL; @TestPropertySource(properties = { "transport.mqtt.enabled=true", @@ -101,6 +100,15 @@ import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest. @ContextConfiguration(classes = {EntityViewControllerTest.Config.class}) @DaoSqlTest public class EntityViewControllerTest extends AbstractControllerTest { + // Must NOT be imported from AbstractMqttIntegrationTest. That field is a static final initialized + // once per JVM. Other test classes (e.g. MqttGatewayRateLimitsTest, DeviceEdgeTest) share the same + // constant but produce a different Spring context cache key, so Spring creates a separate + // ApplicationContext for each of them. Every context starts its own MqttTransportService and tries + // to bind the same port, causing BindException when tests run in the same Surefire JVM fork. + // Declaring the port here gives this context its own independently allocated port. + static final int MQTT_PORT = TestSocketUtils.findAvailableTcpPort(); + static final String MQTT_URL = "tcp://localhost:" + MQTT_PORT; + @DynamicPropertySource static void props(DynamicPropertyRegistry registry) { log.warn("transport.mqtt.bind_port = {}", MQTT_PORT); diff --git a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java index 26705ddacb..671c936585 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java +++ b/application/src/test/java/org/thingsboard/server/controller/TbTestWebSocketClient.java @@ -39,7 +39,6 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityHistoryCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.LatestValueCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.TimeSeriesCmd; -import org.thingsboard.server.service.ws.telemetry.sub.TelemetrySubscriptionUpdate; import java.net.URI; import java.nio.channels.NotYetConnectedException; @@ -53,6 +52,8 @@ public class TbTestWebSocketClient extends WebSocketClient { private static final long TIMEOUT = TimeUnit.SECONDS.toMillis(30); + private final CountDownLatch closeLatch = new CountDownLatch(1); + @Getter private volatile String lastMsg; private volatile CountDownLatch reply; @@ -62,6 +63,7 @@ public class TbTestWebSocketClient extends WebSocketClient { super(serverUri); } + @Override public void onOpen(ServerHandshake serverHandshake) { @@ -69,7 +71,13 @@ public class TbTestWebSocketClient extends WebSocketClient { public void authenticate(String token) { WsCommandsWrapper cmdsWrapper = new WsCommandsWrapper(); - cmdsWrapper.setAuthCmd(new AuthCmd(1, token)); + cmdsWrapper.setAuthCmd(new AuthCmd(1, token, null)); + send(JacksonUtil.toString(cmdsWrapper)); + } + + public void authenticateWithApiKey(String apiKey) { + WsCommandsWrapper cmdsWrapper = new WsCommandsWrapper(); + cmdsWrapper.setAuthCmd(new AuthCmd(1, null, apiKey)); send(JacksonUtil.toString(cmdsWrapper)); } @@ -88,6 +96,16 @@ public class TbTestWebSocketClient extends WebSocketClient { @Override public void onClose(int i, String s, boolean b) { log.info("CLOSED."); + closeLatch.countDown(); + } + + public boolean waitForClose() { + try { + return closeLatch.await(TIMEOUT, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + log.warn("Failed to await close", e); + return false; + } } @Override @@ -275,7 +293,7 @@ public class TbTestWebSocketClient extends WebSocketClient { public JsonNode sendTimeseriesCmd(EntityId entityId, String scope) { log.warn("sendTimeseriesCmd entityId: {}, scope: {}", entityId, scope); - TimeseriesSubscriptionCmd cmd = new TimeseriesSubscriptionCmd(0, 0, 0, 10, null); + TimeseriesSubscriptionCmd cmd = new TimeseriesSubscriptionCmd(0, 0, 0, 10, null); cmd.setEntityId(entityId.getId().toString()); cmd.setEntityType(entityId.getEntityType().toString()); cmd.setCmdId(1); diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebSocketApiTest.java similarity index 78% rename from application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java rename to application/src/test/java/org/thingsboard/server/controller/WebSocketApiTest.java index 87ba0ec3e8..2174b7425f 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebSocketApiTest.java @@ -19,13 +19,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; @@ -60,7 +63,9 @@ import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.dao.nosql.ResultSetSizeLimitExceededException; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.service.subscription.SubscriptionErrorCode; import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -71,13 +76,11 @@ import org.thingsboard.server.service.ws.telemetry.cmd.v2.AlarmStatusUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountCmd; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityCountUpdate; import org.thingsboard.server.service.ws.telemetry.cmd.v2.EntityDataUpdate; -import org.thingsboard.server.service.ws.telemetry.sub.TelemetrySubscriptionUpdate; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -91,10 +94,14 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. "server.ws.alarms_per_alarm_status_subscription_cache_size=5", "server.ws.dynamic_page_link.refresh_interval=15" }) -public class WebsocketApiTest extends AbstractControllerTest { +public class WebSocketApiTest extends AbstractControllerTest { + @Autowired private TelemetrySubscriptionService tsService; + @SpyBean + private TimeseriesService timeseriesService; + Device device; DeviceTypeFilter dtf; @@ -128,8 +135,8 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertEquals(0, pageData.getData().get(0).getTimeseries().get("temperature").length); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertEquals(0, pageData.getData().getFirst().getTimeseries().get("temperature").length); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); TsKvEntry dataPoint2 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(2), new LongDataEntry("temperature", 42L)); @@ -144,8 +151,8 @@ public class WebsocketApiTest extends AbstractControllerTest { List dataList = update.getUpdate(); Assert.assertNotNull(dataList); Assert.assertEquals(1, dataList.size()); - Assert.assertEquals(device.getId(), dataList.get(0).getEntityId()); - TsValue[] tsArray = dataList.get(0).getTimeseries().get("temperature"); + Assert.assertEquals(device.getId(), dataList.getFirst().getEntityId()); + TsValue[] tsArray = dataList.getFirst().getTimeseries().get("temperature"); Assert.assertEquals(3, tsArray.length); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsArray[0]); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsArray[1]); @@ -162,7 +169,7 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); TsKvEntry dataPoint2 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(2), new LongDataEntry("temperature", 43L)); @@ -176,8 +183,8 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - TsValue[] tsArray = listData.get(0).getTimeseries().get("temperature"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + TsValue[] tsArray = listData.getFirst().getTimeseries().get("temperature"); Assert.assertEquals(3, tsArray.length); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsArray[0]); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsArray[1]); @@ -186,7 +193,7 @@ public class WebsocketApiTest extends AbstractControllerTest { now = System.currentTimeMillis(); TsKvEntry dataPoint4 = new BasicTsKvEntry(now, new LongDataEntry("temperature", 45L)); getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint4)); + sendTelemetry(device, List.of(dataPoint4)); String msg = getWsClient().waitForUpdate(); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); @@ -194,9 +201,9 @@ public class WebsocketApiTest extends AbstractControllerTest { List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getTimeseries()); - TsValue[] tsValues = eData.get(0).getTimeseries().get("temperature"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getTimeseries()); + TsValue[] tsValues = eData.getFirst().getTimeseries().get("temperature"); Assert.assertNotNull(tsValues); Assert.assertEquals(new TsValue(dataPoint4.getTs(), dataPoint4.getValueAsString()), tsValues[0]); } @@ -521,7 +528,7 @@ public class WebsocketApiTest extends AbstractControllerTest { String msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); - // check device + // check a device AlarmStatusCmd deviceCmd = new AlarmStatusCmd(2, device.getId(), null, List.of(AlarmSeverity.CRITICAL)); getWsClient().send(deviceCmd); @@ -589,13 +596,13 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendTelemetry(device, tsData); update = getWsClient().subscribeLatestUpdate(keys); @@ -605,36 +612,36 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + Assert.assertNotNull(listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + TsValue tsValue = listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsValue); now = System.currentTimeMillis(); TsKvEntry dataPoint2 = new BasicTsKvEntry(now, new LongDataEntry("temperature", 52L)); getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); Assert.assertEquals(1, update.getCmdId()); List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + tsValue = eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsValue); //Sending update from the past, while latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint1)); + sendTelemetry(device, List.of(dataPoint1)); String msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); } @@ -677,14 +684,14 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); getWsClient().registerWaitForUpdate(); TsKvEntry dataPoint1 = new BasicTsKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("temperature", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendTelemetry(device, tsData); update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); @@ -694,38 +701,73 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + Assert.assertNotNull(listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + TsValue tsValue = listData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint1.getTs(), dataPoint1.getValueAsString()), tsValue); now = System.currentTimeMillis(); TsKvEntry dataPoint2 = new BasicTsKvEntry(now, new LongDataEntry("temperature", 52L)); getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); Assert.assertEquals(1, update.getCmdId()); List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); - tsValue = eData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES)); + tsValue = eData.getFirst().getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); Assert.assertEquals(new TsValue(dataPoint2.getTs(), dataPoint2.getValueAsString()), tsValue); - //Sending update from the past, while latest value has new timestamp; + //Sending update from the past, while the latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint1)); + sendTelemetry(device, List.of(dataPoint1)); String msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendTelemetry(device, Arrays.asList(dataPoint2)); + sendTelemetry(device, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); } + @Test + public void testShouldSendWsUpdateMessageWhenTelemetryWasDeleted() throws Exception { + long now = System.currentTimeMillis() - 100; + TsKvEntry dataPoint = new BasicTsKvEntry(now, new LongDataEntry("temperature", 42L)); + List tsData = List.of(dataPoint); + sendTelemetry(device, tsData); + + List keys = List.of(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataUpdate update = getWsClient().subscribeLatestUpdate(keys, dtf); + + Assert.assertEquals(1, update.getCmdId()); + PageData pageData = update.getData(); + Assert.assertNotNull(pageData); + Assert.assertEquals(1, pageData.getData().size()); + Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); + Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); + Assert.assertEquals(now, pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); + Assert.assertEquals("42", pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); + + // delete telemetry + getWsClient().registerWaitForUpdate(); + doDeleteAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/delete?keys=temperature&deleteAllDataForKeys=true", String.class); + update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); + + Assert.assertEquals(1, update.getCmdId()); + + List listData = update.getUpdate(); + Assert.assertNotNull(listData); + Assert.assertEquals(1, listData.size()); + Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); + Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); + TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(new TsValue(0, ""), tsValue); + } + @Test public void testEntityDataLatestAttrWsCmd() throws Exception { long now = System.currentTimeMillis(); @@ -736,21 +778,21 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); getWsClient().registerWaitForUpdate(); - // Pushing update with wrong scope and make sure it will not arrive. + // Pushing update with the wrong scope and make sure it will not arrive. AttributeKvEntry invalidDataPoint = new BaseAttributeKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("serverAttributeKey", 55L)); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(invalidDataPoint)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(invalidDataPoint)); Assert.assertNull(getWsClient().waitForUpdate(3000)); AttributeKvEntry dataPoint1 = new BaseAttributeKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("serverAttributeKey", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, tsData); String msg = getWsClient().waitForUpdate(); @@ -761,16 +803,16 @@ public class WebsocketApiTest extends AbstractControllerTest { List listData = update.getUpdate(); Assert.assertNotNull(listData); Assert.assertEquals(1, listData.size()); - Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); - Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); - TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); + Assert.assertEquals(device.getId(), listData.getFirst().getEntityId()); + Assert.assertNotNull(listData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); + TsValue tsValue = listData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); Assert.assertEquals(new TsValue(dataPoint1.getLastUpdateTs(), dataPoint1.getValueAsString()), tsValue); now = System.currentTimeMillis(); AttributeKvEntry dataPoint2 = new BaseAttributeKvEntry(now, new LongDataEntry("serverAttributeKey", 52L)); getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint2)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(); Assert.assertNotNull(msg); @@ -779,20 +821,20 @@ public class WebsocketApiTest extends AbstractControllerTest { List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); - tsValue = eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); + tsValue = eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); Assert.assertEquals(new TsValue(dataPoint2.getLastUpdateTs(), dataPoint2.getValueAsString()), tsValue); - //Sending update from the past, while latest value has new timestamp; + //Sending update from the past, while the latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint1)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint1)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint2)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); } @@ -812,23 +854,23 @@ public class WebsocketApiTest extends AbstractControllerTest { PageData pageData = update.getData(); Assert.assertNotNull(pageData); Assert.assertEquals(1, pageData.getData().size()); - Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getValue()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getValue()); - Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey")); - Assert.assertEquals(0, pageData.getData().get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getTs()); - Assert.assertEquals("", pageData.getData().get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getValue()); + Assert.assertEquals(device.getId(), pageData.getData().getFirst().getEntityId()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey").getValue()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey").getValue()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey").getValue()); + Assert.assertNotNull(pageData.getData().getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey")); + Assert.assertEquals(0, pageData.getData().getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getTs()); + Assert.assertEquals("", pageData.getData().getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey").getValue()); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint1 = new BaseAttributeKvEntry(now - TimeUnit.MINUTES.toMillis(1), new LongDataEntry("serverAttributeKey", 42L)); - List tsData = Arrays.asList(dataPoint1); + List tsData = List.of(dataPoint1); sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, tsData); @@ -839,78 +881,78 @@ public class WebsocketApiTest extends AbstractControllerTest { List eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); - TsValue attrValue = eData.get(0).getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE)); + TsValue attrValue = eData.getFirst().getLatest().get(EntityKeyType.SERVER_ATTRIBUTE).get("serverAttributeKey"); Assert.assertEquals(new TsValue(dataPoint1.getLastUpdateTs(), dataPoint1.getValueAsString()), attrValue); - //Sending update from the past, while latest value has new timestamp; + //Sending update from the past, while the latest value has new timestamp; getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, Arrays.asList(dataPoint1)); + sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, List.of(dataPoint1)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending duplicate update again getWsClient().registerWaitForUpdate(); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(dataPoint1)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(dataPoint1)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); Assert.assertNull(msg); //Sending update from the past, while latest value has new timestamp; getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint2 = new BaseAttributeKvEntry(now, new LongDataEntry("sharedAttributeKey", 42L)); - sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, Arrays.asList(dataPoint2)); + sendAttributes(device, TbAttributeSubscriptionScope.SHARED_SCOPE, List.of(dataPoint2)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.SHARED_ATTRIBUTE).get("sharedAttributeKey"); Assert.assertEquals(new TsValue(dataPoint2.getLastUpdateTs(), dataPoint2.getValueAsString()), attrValue); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint3 = new BaseAttributeKvEntry(now, new LongDataEntry("clientAttributeKey", 42L)); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(dataPoint3)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(dataPoint3)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.CLIENT_ATTRIBUTE).get("clientAttributeKey"); Assert.assertEquals(new TsValue(dataPoint3.getLastUpdateTs(), dataPoint3.getValueAsString()), attrValue); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint4 = new BaseAttributeKvEntry(now, new LongDataEntry("anyAttributeKey", 42L)); - sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, Arrays.asList(dataPoint4)); + sendAttributes(device, TbAttributeSubscriptionScope.CLIENT_SCOPE, List.of(dataPoint4)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); Assert.assertEquals(new TsValue(dataPoint4.getLastUpdateTs(), dataPoint4.getValueAsString()), attrValue); getWsClient().registerWaitForUpdate(); AttributeKvEntry dataPoint5 = new BaseAttributeKvEntry(now, new LongDataEntry("anyAttributeKey", 43L)); - sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, Arrays.asList(dataPoint5)); + sendAttributes(device, TbAttributeSubscriptionScope.SERVER_SCOPE, List.of(dataPoint5)); msg = getWsClient().waitForUpdate(TimeUnit.SECONDS.toMillis(1)); update = JacksonUtil.fromString(msg, EntityDataUpdate.class); Assert.assertEquals(1, update.getCmdId()); eData = update.getUpdate(); Assert.assertNotNull(eData); Assert.assertEquals(1, eData.size()); - Assert.assertEquals(device.getId(), eData.get(0).getEntityId()); - Assert.assertNotNull(eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE)); - attrValue = eData.get(0).getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); + Assert.assertEquals(device.getId(), eData.getFirst().getEntityId()); + Assert.assertNotNull(eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE)); + attrValue = eData.getFirst().getLatest().get(EntityKeyType.ATTRIBUTE).get("anyAttributeKey"); Assert.assertEquals(new TsValue(dataPoint5.getLastUpdateTs(), dataPoint5.getValueAsString()), attrValue); } @@ -965,13 +1007,41 @@ public class WebsocketApiTest extends AbstractControllerTest { } + @Test + public void testHistoryCmdSendsWsErrorOnResultSetSizeLimitExceeded() throws Exception { + ResultSetSizeLimitExceededException exception = new ResultSetSizeLimitExceededException(100L, 200L); + Mockito.doReturn(Futures.immediateFailedFuture(exception)) + .when(timeseriesService).findAllByQueries(Mockito.any(), Mockito.any(), Mockito.any()); + + List keys = List.of("temperature"); + long now = System.currentTimeMillis(); + + EntityDataUpdate errorUpdate = getWsClient().sendHistoryCmd(keys, now, TimeUnit.HOURS.toMillis(1), dtf); + assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); + assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); + } + + @Test + public void testTimeSeriesCmdSendsWsErrorOnResultSetSizeLimitExceeded() throws Exception { + ResultSetSizeLimitExceededException exception = new ResultSetSizeLimitExceededException(100L, 200L); + Mockito.doReturn(Futures.immediateFailedFuture(exception)) + .when(timeseriesService).findAllByQueries(Mockito.any(), Mockito.any(), Mockito.any()); + + List keys = List.of("temperature"); + long now = System.currentTimeMillis(); + + EntityDataUpdate errorUpdate = getWsClient().subscribeTsUpdate(keys, now, TimeUnit.HOURS.toMillis(1), dtf); + assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); + assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); + } + private void sendTelemetry(Device device, List tsData) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); tsService.saveTimeseries(TimeseriesSaveRequest.builder() .tenantId(device.getTenantId()) .entityId(device.getId()) .entries(tsData) - .callback(new FutureCallback() { + .callback(new FutureCallback<>() { @Override public void onSuccess(@Nullable Void result) { log.debug("sendTelemetry callback onSuccess"); diff --git a/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java b/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java index 053cb6808f..accbbd1d29 100644 --- a/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/plugin/TbWebSocketHandlerTest.java @@ -25,16 +25,25 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import org.springframework.test.util.ReflectionTestUtils; import org.springframework.web.socket.CloseStatus; import org.springframework.web.socket.adapter.NativeWebSocketSession; import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; +import org.thingsboard.server.service.security.auth.pat.ApiKeyAuthenticationProvider; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.ws.WebSocketService; import org.thingsboard.server.service.ws.WebSocketSessionRef; +import org.thingsboard.server.service.ws.WebSocketSessionType; import java.io.IOException; +import java.lang.reflect.Method; import java.util.Collection; import java.util.Deque; import java.util.List; import java.util.Random; +import java.util.UUID; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CountDownLatch; @@ -184,4 +193,101 @@ class TbWebSocketHandlerTest { assertThat(msgs).map(Integer::parseInt).doesNotHaveDuplicates().hasSize(100); } + private AuthTestFixture createAuthTestFixture() throws IOException { + TbWebSocketHandler handler = spy(new TbWebSocketHandler()); + willDoNothing().given(handler).close(any(), any()); + + ApiKeyAuthenticationProvider apiKeyProvider = mock(ApiKeyAuthenticationProvider.class); + JwtAuthenticationProvider jwtProvider = mock(JwtAuthenticationProvider.class); + WebSocketService wsService = mock(WebSocketService.class); + + ReflectionTestUtils.setField(handler, "apiKeyAuthenticationProvider", apiKeyProvider); + ReflectionTestUtils.setField(handler, "authenticationProvider", jwtProvider); + ReflectionTestUtils.setField(handler, "webSocketService", wsService); + ReflectionTestUtils.setField(handler, "tenantProfileCache", mock(TbTenantProfileCache.class)); + ReflectionTestUtils.setField(handler, "authTimeoutMs", 10000); + ReflectionTestUtils.invokeMethod(handler, "init"); + + WebSocketSessionRef ref = WebSocketSessionRef.builder() + .sessionId(UUID.randomUUID().toString()) + .sessionType(WebSocketSessionType.GENERAL) + .build(); + + NativeWebSocketSession wsSession = mock(NativeWebSocketSession.class); + Session nativeSess = mock(Session.class); + willReturn(nativeSess).given(wsSession).getNativeSession(Session.class); + RemoteEndpoint.Async async = mock(RemoteEndpoint.Async.class); + willReturn(async).given(nativeSess).getAsyncRemote(); + willReturn("test-session-id").given(wsSession).getId(); + + TbWebSocketHandler.SessionMetaData sessionMd = handler.new SessionMetaData(wsSession, ref); + + return new AuthTestFixture(handler, apiKeyProvider, jwtProvider, ref, sessionMd); + } + + @Test + void processMsg_authenticatesWithApiKey() throws Exception { + AuthTestFixture f = createAuthTestFixture(); + + SecurityUser securityUser = mock(SecurityUser.class, Mockito.RETURNS_DEEP_STUBS); + willReturn(securityUser).given(f.apiKeyProvider).authenticate("my-api-key"); + + String msg = "{\"authCmd\":{\"cmdId\":1,\"apiKey\":\"my-api-key\"},\"cmds\":[]}"; + f.handler.processMsg(f.sessionMd, msg); + + verify(f.apiKeyProvider).authenticate("my-api-key"); + verify(f.jwtProvider, never()).authenticate(anyString()); + assertThat(f.ref.getSecurityCtx()).isSameAs(securityUser); + } + + @Test + void processMsg_authenticatesWithJwtToken() throws Exception { + AuthTestFixture f = createAuthTestFixture(); + + SecurityUser securityUser = mock(SecurityUser.class, Mockito.RETURNS_DEEP_STUBS); + willReturn(securityUser).given(f.jwtProvider).authenticate("my-jwt-token"); + + String msg = "{\"authCmd\":{\"cmdId\":1,\"token\":\"my-jwt-token\"},\"cmds\":[]}"; + f.handler.processMsg(f.sessionMd, msg); + + verify(f.jwtProvider).authenticate("my-jwt-token"); + verify(f.apiKeyProvider, never()).authenticate(anyString()); + assertThat(f.ref.getSecurityCtx()).isSameAs(securityUser); + } + + @Test + void processMsg_apiKeyTakesPrecedenceOverToken() throws Exception { + AuthTestFixture f = createAuthTestFixture(); + + SecurityUser securityUser = mock(SecurityUser.class, Mockito.RETURNS_DEEP_STUBS); + willReturn(securityUser).given(f.apiKeyProvider).authenticate("my-api-key"); + + String msg = "{\"authCmd\":{\"cmdId\":1,\"apiKey\":\"my-api-key\",\"token\":\"my-jwt-token\"},\"cmds\":[]}"; + f.handler.processMsg(f.sessionMd, msg); + + verify(f.apiKeyProvider).authenticate("my-api-key"); + verify(f.jwtProvider, never()).authenticate(anyString()); + assertThat(f.ref.getSecurityCtx()).isSameAs(securityUser); + } + + @Test + void extractQueryParam_parsesCorrectly() throws Exception { + TbWebSocketHandler handler = new TbWebSocketHandler(); + Method method = TbWebSocketHandler.class.getDeclaredMethod("extractQueryParam", String.class, String.class); + method.setAccessible(true); + + assertThat(method.invoke(handler, "token=jwt123", "token")).isEqualTo("jwt123"); + assertThat(method.invoke(handler, "token=jwt123&other=abc123", "token")).isEqualTo("jwt123"); + assertThat(method.invoke(handler, "other=value", "token")).isNull(); + assertThat(method.invoke(handler, "tokenExtra=value", "token")).isNull(); + } + + private record AuthTestFixture( + TbWebSocketHandler handler, + ApiKeyAuthenticationProvider apiKeyProvider, + JwtAuthenticationProvider jwtProvider, + WebSocketSessionRef ref, + TbWebSocketHandler.SessionMetaData sessionMd + ) {} + } diff --git a/application/src/test/java/org/thingsboard/server/edge/ApiKeyEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/ApiKeyEdgeTest.java new file mode 100644 index 0000000000..bd46cb5251 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/edge/ApiKeyEdgeTest.java @@ -0,0 +1,238 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.edge; + +import com.datastax.oss.driver.api.core.uuid.Uuids; +import com.google.protobuf.AbstractMessage; +import com.google.protobuf.InvalidProtocolBufferException; +import org.junit.Assert; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.User; +import org.thingsboard.server.common.data.id.ApiKeyId; +import org.thingsboard.server.common.data.pat.ApiKey; +import org.thingsboard.server.common.data.pat.ApiKeyInfo; +import org.thingsboard.server.common.data.security.Authority; +import org.thingsboard.server.dao.pat.ApiKeyService; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UpdateMsgType; +import org.thingsboard.server.gen.edge.v1.UplinkMsg; +import org.thingsboard.server.gen.edge.v1.UplinkResponseMsg; +import org.thingsboard.server.gen.edge.v1.UserCredentialsUpdateMsg; +import org.thingsboard.server.gen.edge.v1.UserUpdateMsg; + +import java.util.Optional; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.awaitility.Awaitility.await; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; +import static org.thingsboard.server.gen.edge.v1.UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE; + +@DaoSqlTest +public class ApiKeyEdgeTest extends AbstractEdgeTest { + + @Autowired + private ApiKeyService apiKeyService; + + private static final String DEFAULT_API_KEY_DESCRIPTION = "Edge Test ApiKey"; + private static final String UPDATED_API_KEY_DESCRIPTION = "Updated Edge Test ApiKey"; + + @Test + public void testApiKey_create_update_delete_fromCloud() throws Exception { + // create ApiKey + ApiKeyInfo apiKeyInfo = createSimpleApiKeyInfo(DEFAULT_API_KEY_DESCRIPTION); + + edgeImitator.expectMessageAmount(1); + ApiKey savedApiKey = doPost("/api/apiKey", apiKeyInfo, ApiKey.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + + AbstractMessage latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof ApiKeyUpdateMsg); + ApiKeyUpdateMsg apiKeyUpdateMsg = (ApiKeyUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + Assert.assertEquals(savedApiKey.getUuidId().getMostSignificantBits(), apiKeyUpdateMsg.getIdMSB()); + Assert.assertEquals(savedApiKey.getUuidId().getLeastSignificantBits(), apiKeyUpdateMsg.getIdLSB()); + ApiKey apiKeyFromMsg = JacksonUtil.fromString(apiKeyUpdateMsg.getEntity(), ApiKey.class, true); + Assert.assertNotNull(apiKeyFromMsg); + + Assert.assertEquals(DEFAULT_API_KEY_DESCRIPTION, apiKeyFromMsg.getDescription()); + Assert.assertEquals(savedApiKey.getTenantId(), apiKeyFromMsg.getTenantId()); + + // update ApiKey + edgeImitator.expectMessageAmount(1); + savedApiKey.setDescription(UPDATED_API_KEY_DESCRIPTION); + savedApiKey = doPost("/api/apiKey", new ApiKeyInfo(savedApiKey), ApiKey.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + + latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof ApiKeyUpdateMsg); + apiKeyUpdateMsg = (ApiKeyUpdateMsg) latestMessage; + apiKeyFromMsg = JacksonUtil.fromString(apiKeyUpdateMsg.getEntity(), ApiKey.class, true); + Assert.assertNotNull(apiKeyFromMsg); + Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + Assert.assertEquals(UPDATED_API_KEY_DESCRIPTION, apiKeyFromMsg.getDescription()); + + // delete ApiKey + edgeImitator.expectMessageAmount(1); + doDelete("/api/apiKey/" + savedApiKey.getUuidId()) + .andExpect(status().isOk()); + Assert.assertTrue(edgeImitator.waitForMessages()); + + latestMessage = edgeImitator.getLatestMessage(); + Assert.assertTrue(latestMessage instanceof ApiKeyUpdateMsg); + apiKeyUpdateMsg = (ApiKeyUpdateMsg) latestMessage; + Assert.assertEquals(UpdateMsgType.ENTITY_DELETED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + Assert.assertEquals(savedApiKey.getUuidId().getMostSignificantBits(), apiKeyUpdateMsg.getIdMSB()); + Assert.assertEquals(savedApiKey.getUuidId().getLeastSignificantBits(), apiKeyUpdateMsg.getIdLSB()); + } + + @Test + public void testApiKey_create_update_delete_toCloud() throws Exception { + // create + ApiKey apiKey = createSimpleApiKey(DEFAULT_API_KEY_DESCRIPTION); + UUID uuid = Uuids.timeBased(); + UplinkMsg uplinkMsg = getUplinkMsg(uuid, apiKey, UpdateMsgType.ENTITY_CREATED_RPC_MESSAGE); + + checkApiKeyOnCloud(uplinkMsg, uuid, apiKey.getDescription()); + + // update + apiKey.setDescription(UPDATED_API_KEY_DESCRIPTION); + UplinkMsg updatedUplinkMsg = getUplinkMsg(uuid, apiKey, UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE); + + checkApiKeyOnCloud(updatedUplinkMsg, uuid, apiKey.getDescription()); + + // delete + UplinkMsg deleteUplinkMsg = getDeleteUplinkMsg(uuid); + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(deleteUplinkMsg); + Assert.assertTrue(edgeImitator.waitForResponses()); + + ApiKeyId apiKeyId = new ApiKeyId(uuid); + await().atMost(30, TimeUnit.SECONDS).untilAsserted(() -> + Assert.assertNull(apiKeyService.findApiKeyById(tenantId, apiKeyId)) + ); + } + + @Test + public void testApiKey_pushedDuringUserSync() throws Exception { + // create tenant admin user - expect 3 messages: 1 UserUpdateMsg + 2 UserCredentialsUpdateMsg + User user = new User(); + user.setAuthority(Authority.TENANT_ADMIN); + user.setTenantId(tenantId); + user.setEmail("apiKeyTestUser@thingsboard.org"); + user.setFirstName("ApiKey"); + user.setLastName("TestUser"); + + edgeImitator.expectMessageAmount(3); + User savedUser = createUser(user, "tenant"); + Assert.assertTrue(edgeImitator.waitForMessages()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(UserUpdateMsg.class).size()); + Assert.assertEquals(2, edgeImitator.findAllMessagesByType(UserCredentialsUpdateMsg.class).size()); + + // create API key for this user - expect 1 ApiKeyUpdateMsg + ApiKeyInfo apiKeyInfo = new ApiKeyInfo(); + apiKeyInfo.setTenantId(tenantId); + apiKeyInfo.setUserId(savedUser.getId()); + apiKeyInfo.setDescription("Test API Key for user sync"); + apiKeyInfo.setEnabled(true); + + edgeImitator.expectMessageAmount(1); + doPost("/api/apiKey", apiKeyInfo, ApiKey.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(ApiKeyUpdateMsg.class).size()); + + // update user - expect 3 messages: UserUpdateMsg + UserCredentialsUpdateMsg + ApiKeyUpdateMsg + savedUser.setLastName("UpdatedLastName"); + edgeImitator.expectMessageAmount(3); + doPost("/api/user", savedUser, User.class); + Assert.assertTrue(edgeImitator.waitForMessages()); + + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(UserUpdateMsg.class).size()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(UserCredentialsUpdateMsg.class).size()); + Assert.assertEquals(1, edgeImitator.findAllMessagesByType(ApiKeyUpdateMsg.class).size()); + + Optional apiKeyUpdateMsgOpt = edgeImitator.findMessageByType(ApiKeyUpdateMsg.class); + Assert.assertTrue(apiKeyUpdateMsgOpt.isPresent()); + ApiKeyUpdateMsg apiKeyUpdateMsg = apiKeyUpdateMsgOpt.get(); + Assert.assertEquals(UpdateMsgType.ENTITY_UPDATED_RPC_MESSAGE, apiKeyUpdateMsg.getMsgType()); + } + + private ApiKeyInfo createSimpleApiKeyInfo(String description) { + ApiKeyInfo apiKeyInfo = new ApiKeyInfo(); + apiKeyInfo.setTenantId(tenantId); + apiKeyInfo.setUserId(tenantAdminUserId); + apiKeyInfo.setDescription(description); + apiKeyInfo.setEnabled(true); + return apiKeyInfo; + } + + private ApiKey createSimpleApiKey(String description) { + ApiKey apiKey = new ApiKey(); + apiKey.setTenantId(tenantId); + apiKey.setUserId(tenantAdminUserId); + apiKey.setDescription(description); + apiKey.setEnabled(true); + apiKey.setValue("test-api-key-value-" + UUID.randomUUID()); + return apiKey; + } + + private UplinkMsg getDeleteUplinkMsg(UUID uuid) throws InvalidProtocolBufferException { + UplinkMsg.Builder upLinkMsgBuilder = UplinkMsg.newBuilder(); + ApiKeyUpdateMsg.Builder apiKeyDeleteMsgBuilder = ApiKeyUpdateMsg.newBuilder(); + apiKeyDeleteMsgBuilder.setMsgType(ENTITY_DELETED_RPC_MESSAGE); + apiKeyDeleteMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + apiKeyDeleteMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + testAutoGeneratedCodeByProtobuf(apiKeyDeleteMsgBuilder); + + upLinkMsgBuilder.addApiKeyUpdateMsg(apiKeyDeleteMsgBuilder.build()); + testAutoGeneratedCodeByProtobuf(upLinkMsgBuilder); + + return upLinkMsgBuilder.build(); + } + + private UplinkMsg getUplinkMsg(UUID uuid, ApiKey apiKey, UpdateMsgType updateMsgType) throws InvalidProtocolBufferException { + UplinkMsg.Builder uplinkMsgBuilder = UplinkMsg.newBuilder(); + ApiKeyUpdateMsg.Builder apiKeyUpdateMsgBuilder = ApiKeyUpdateMsg.newBuilder(); + apiKeyUpdateMsgBuilder.setIdMSB(uuid.getMostSignificantBits()); + apiKeyUpdateMsgBuilder.setIdLSB(uuid.getLeastSignificantBits()); + apiKeyUpdateMsgBuilder.setEntity(JacksonUtil.toString(apiKey)); + apiKeyUpdateMsgBuilder.setMsgType(updateMsgType); + testAutoGeneratedCodeByProtobuf(apiKeyUpdateMsgBuilder); + uplinkMsgBuilder.addApiKeyUpdateMsg(apiKeyUpdateMsgBuilder.build()); + + testAutoGeneratedCodeByProtobuf(uplinkMsgBuilder); + + return uplinkMsgBuilder.build(); + } + + private void checkApiKeyOnCloud(UplinkMsg uplinkMsg, UUID uuid, String description) throws Exception { + edgeImitator.expectResponsesAmount(1); + edgeImitator.sendUplinkMsg(uplinkMsg); + + Assert.assertTrue(edgeImitator.waitForResponses()); + + UplinkResponseMsg latestResponseMsg = edgeImitator.getLatestResponseMsg(); + Assert.assertTrue(latestResponseMsg.getSuccess()); + + ApiKey apiKey = apiKeyService.findApiKeyById(tenantId, new ApiKeyId(uuid)); + Assert.assertNotNull(apiKey); + Assert.assertEquals(description, apiKey.getDescription()); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/edge/CalculatedFieldEdgeTest.java b/application/src/test/java/org/thingsboard/server/edge/CalculatedFieldEdgeTest.java index 60701cee1a..52a30f0e85 100644 --- a/application/src/test/java/org/thingsboard/server/edge/CalculatedFieldEdgeTest.java +++ b/application/src/test/java/org/thingsboard/server/edge/CalculatedFieldEdgeTest.java @@ -73,6 +73,7 @@ public class CalculatedFieldEdgeTest extends AbstractEdgeTest { Assert.assertEquals(DEFAULT_CF_NAME, calculatedFieldFromMsg.getName()); Assert.assertEquals(savedDevice.getId(), calculatedFieldFromMsg.getEntityId()); Assert.assertEquals(config, calculatedFieldFromMsg.getConfiguration()); + Assert.assertEquals(calculatedField.getAdditionalInfo(), calculatedFieldFromMsg.getAdditionalInfo()); edgeImitator.expectMessageAmount(1); savedCalculatedField.setName(UPDATED_CF_NAME); @@ -229,6 +230,7 @@ public class CalculatedFieldEdgeTest extends AbstractEdgeTest { config.setOutput(output); calculatedField.setConfiguration(config); + calculatedField.setAdditionalInfo(JacksonUtil.newObjectNode()); return calculatedField; } @@ -260,6 +262,7 @@ public class CalculatedFieldEdgeTest extends AbstractEdgeTest { CalculatedField calculatedField = doGet("/api/calculatedField/" + uuid, CalculatedField.class); Assert.assertNotNull(calculatedField); Assert.assertEquals(resourceTitle, calculatedField.getName()); + Assert.assertEquals(JacksonUtil.newObjectNode(), calculatedField.getAdditionalInfo()); } } diff --git a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java index 8cf2b309f6..8208dc4fc9 100644 --- a/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java +++ b/application/src/test/java/org/thingsboard/server/edge/imitator/EdgeImitator.java @@ -30,6 +30,7 @@ import org.thingsboard.edge.rpc.EdgeRpcClient; import org.thingsboard.server.controller.AbstractWebTest; import org.thingsboard.server.gen.edge.v1.AdminSettingsUpdateMsg; import org.thingsboard.server.gen.edge.v1.AiModelUpdateMsg; +import org.thingsboard.server.gen.edge.v1.ApiKeyUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmCommentUpdateMsg; import org.thingsboard.server.gen.edge.v1.AlarmUpdateMsg; import org.thingsboard.server.gen.edge.v1.AssetProfileUpdateMsg; @@ -375,6 +376,11 @@ public class EdgeImitator { result.add(saveDownlinkMsg(aiModelUpdateMsg)); } } + if (downlinkMsg.getApiKeyUpdateMsgCount() > 0) { + for (ApiKeyUpdateMsg apiKeyUpdateMsg : downlinkMsg.getApiKeyUpdateMsgList()) { + result.add(saveDownlinkMsg(apiKeyUpdateMsg)); + } + } if (downlinkMsg.hasEdgeConfiguration()) { result.add(saveDownlinkMsg(downlinkMsg.getEdgeConfiguration())); } diff --git a/application/src/test/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSslTest.java b/application/src/test/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSslTest.java new file mode 100644 index 0000000000..04a1b87833 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSslTest.java @@ -0,0 +1,272 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc; + +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.openssl.jcajce.JcePEMEncryptorBuilder; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.bouncycastle.util.io.pem.PemObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.controller.AbstractWebTest; +import org.thingsboard.server.gen.edge.v1.EdgeRpcServiceGrpc; + +import java.io.ByteArrayInputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.X509Certificate; +import java.security.spec.ECGenParameterSpec; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; + +/** + * Tests for Edge gRPC SSL setup using the production {@link EdgeGrpcService#setupSsl} method. + *

+ * Covers: + * 1. Separate cert and key PEM inputs + * 2. Combined PEM (cert + key in one file) + * 3. Encrypted private key with password + * 4. Missing key in combined PEM → error + *

+ * Each scenario is parameterized across key types: RSA-2048, RSA-4096, EC P-256, EC P-384. + */ +class EdgeGrpcSslTest { + + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + enum KeyType { + RSA_2048("RSA", 2048, null, "SHA256withRSA"), + RSA_4096("RSA", 4096, null, "SHA256withRSA"), + EC_P256("EC", 256, "secp256r1", "SHA256withECDSA"), + EC_P384("EC", 384, "secp384r1", "SHA384withECDSA"); + + final String algorithm; + final int size; + final String curve; + final String sigAlg; + + KeyType(String algorithm, int size, String curve, String sigAlg) { + this.algorithm = algorithm; + this.size = size; + this.curve = curve; + this.sigAlg = sigAlg; + } + + KeyPair generateKeyPair() throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance(algorithm); + if (curve != null) { + kpg.initialize(new ECGenParameterSpec(curve)); + } else { + kpg.initialize(size); + } + return kpg.generateKeyPair(); + } + } + + private final List tempFiles = new ArrayList<>(); + private Server server; + private ManagedChannel channel; + + @AfterEach + void cleanup() throws Exception { + if (channel != null) { + channel.shutdownNow().awaitTermination(2, TimeUnit.SECONDS); + } + if (server != null) { + server.shutdownNow().awaitTermination(2, TimeUnit.SECONDS); + } + for (Path p : tempFiles) { + Files.deleteIfExists(p); + } + } + + @ParameterizedTest(name = "separateCertAndKey_{0}") + @EnumSource(KeyType.class) + void separateCertAndKey(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + + Path certFile = writeTempPem("cert", cert); + Path keyFile = writeTempPem("key", kp.getPrivate()); + + server = startServer(certFile.toString(), keyFile.toString(), null); + assertTlsConnectivity(cert); + } + + @ParameterizedTest(name = "combinedPemWithCertAndKey_{0}") + @EnumSource(KeyType.class) + void combinedPemWithCertAndKey(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + + Path combinedFile = writeTempPem("combined", cert, kp.getPrivate()); + + server = startServer(combinedFile.toString(), "", null); + assertTlsConnectivity(cert); + } + + // RSA-only: BouncyCastle writes encrypted EC keys in traditional PEM format (BEGIN EC PRIVATE KEY), + // which after decryption produces a PEMKeyPair without public key info — causing PemSslCredentials + // to fail with "Cannot invoke SubjectPublicKeyInfo.getEncoded() because getPublicKeyInfo() is null". + @ParameterizedTest(name = "encryptedPrivateKey_{0}") + @EnumSource(value = KeyType.class, names = {"RSA_2048", "RSA_4096"}) + void encryptedPrivateKey(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + String password = "test-password"; + + Path combinedFile = writeTempPemEncrypted("enc-combined", password, cert, kp.getPrivate()); + + server = startServer(combinedFile.toString(), "", password); + assertTlsConnectivity(cert); + } + + @ParameterizedTest(name = "combinedPemWithCertOnly_throwsException_{0}") + @EnumSource(KeyType.class) + void combinedPemWithCertOnly_throwsException(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + + Path certOnlyFile = writeTempPem("cert-only", cert); + + assertThatThrownBy(() -> startServer(certOnlyFile.toString(), "", null)) + .isInstanceOf(IllegalArgumentException.class); + } + + // --- Server startup using production EdgeGrpcService.setupSsl() --- + + private Server startServer(String certFileResource, String privateKeyResource, String keyPassword) throws Exception { + EdgeGrpcService edgeGrpcService = new EdgeGrpcService(); + ReflectionTestUtils.setField(edgeGrpcService, "certFileResource", certFileResource); + ReflectionTestUtils.setField(edgeGrpcService, "privateKeyResource", privateKeyResource); + ReflectionTestUtils.setField(edgeGrpcService, "keyPassword", keyPassword != null ? keyPassword : ""); + + NettyServerBuilder builder = NettyServerBuilder.forPort(0) + .addService(new EdgeRpcServiceGrpc.EdgeRpcServiceImplBase() {}); + + edgeGrpcService.setupSsl(builder); + + return builder.build().start(); + } + + private void assertTlsConnectivity(X509Certificate trustedCert) throws Exception { + String certPem = toPem(trustedCert); + var clientSsl = GrpcSslContexts.forClient() + .trustManager(new ByteArrayInputStream(certPem.getBytes(StandardCharsets.UTF_8))) + .build(); + + channel = NettyChannelBuilder.forAddress("localhost", server.getPort()) + .sslContext(clientSsl) + .build(); + + channel.getState(true); // trigger connection attempt + await().atMost(AbstractWebTest.TIMEOUT, TimeUnit.SECONDS) + .pollInterval(50, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + var state = channel.getState(false); + if (state == io.grpc.ConnectivityState.TRANSIENT_FAILURE) { + throw new AssertionError("TLS handshake failed: channel in TRANSIENT_FAILURE"); + } + assertThat(state).isEqualTo(io.grpc.ConnectivityState.READY); + }); + } + + // --- Cert/key generation --- + + private X509Certificate generateSelfSignedCert(KeyPair kp, String sigAlg) throws Exception { + X500Name subject = new X500Name("CN=localhost"); + Date now = new Date(); + return new JcaX509CertificateConverter().getCertificate( + new JcaX509v3CertificateBuilder( + subject, BigInteger.ONE, now, + new Date(now.getTime() + TimeUnit.DAYS.toMillis(1)), + subject, kp.getPublic()) + .build(new JcaContentSignerBuilder(sigAlg).build(kp.getPrivate()))); + } + + // --- PEM file helpers --- + + private String toPem(Object obj) throws Exception { + java.io.StringWriter sw = new java.io.StringWriter(); + try (JcaPEMWriter w = new JcaPEMWriter(sw)) { + w.writeObject(obj); + } + return sw.toString(); + } + + private Path writeTempPem(String prefix, Object... objects) throws Exception { + Path p = Files.createTempFile(prefix + "-", ".pem"); + tempFiles.add(p); + try (JcaPEMWriter w = new JcaPEMWriter(Files.newBufferedWriter(p))) { + for (Object o : objects) { + w.writeObject(toPkcs8IfKey(o)); + } + } + return p; + } + + private Path writeTempPemEncrypted(String prefix, String password, Object... objects) throws Exception { + Path p = Files.createTempFile(prefix + "-", ".pem"); + tempFiles.add(p); + var encryptor = new JcePEMEncryptorBuilder("AES-256-CBC") + .setProvider(BouncyCastleProvider.PROVIDER_NAME) + .build(password.toCharArray()); + try (JcaPEMWriter w = new JcaPEMWriter(Files.newBufferedWriter(p))) { + for (Object o : objects) { + if (o instanceof PrivateKey) { + w.writeObject(o, encryptor); + } else { + w.writeObject(o); + } + } + } + return p; + } + + private Object toPkcs8IfKey(Object o) { + if (o instanceof PrivateKey pk) { + return new PemObject("PRIVATE KEY", pk.getEncoded()); + } + return o; + } +} diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index c54ffe4a07..05ab1165f6 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -115,6 +115,8 @@ import java.util.Map; import java.util.Random; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -1749,13 +1751,13 @@ public class EntityServiceTest extends AbstractControllerTest { } @Test - public void testFindTenantTelemetry() { + public void testFindTenantTelemetry() throws ExecutionException, InterruptedException, TimeoutException { // save timeseries by sys admin BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, new DoubleDataEntry("temperature", 45.5)); - timeseriesService.save(TenantId.SYS_TENANT_ID, tenantId, timeseries); + timeseriesService.save(TenantId.SYS_TENANT_ID, tenantId, timeseries).get(TIMEOUT, TimeUnit.SECONDS); AttributeKvEntry attr = new BaseAttributeKvEntry(new LongDataEntry("attr", 10L), 42L); - attributesService.save(TenantId.SYS_TENANT_ID, tenantId, SERVER_SCOPE, List.of(attr)); + attributesService.save(TenantId.SYS_TENANT_ID, tenantId, SERVER_SCOPE, List.of(attr)).get(TIMEOUT, TimeUnit.SECONDS); SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); singleEntityFilter.setSingleEntity(AliasEntityId.fromEntityId(tenantId)); diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index cf37afad5c..0c12767b60 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -23,8 +23,8 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; @@ -127,10 +127,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. }) public class HousekeeperServiceTest extends AbstractControllerTest { - @SpyBean + @MockitoSpyBean private HousekeeperService housekeeperService; - @SpyBean + @MockitoSpyBean private HousekeeperReprocessingService housekeeperReprocessingService; + @MockitoSpyBean + private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; @Autowired private EventService eventService; @Autowired @@ -153,8 +155,6 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private CustomerService customerService; @Autowired private DashboardService dashboardService; - @SpyBean - private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; private TenantId tenantId; diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java new file mode 100644 index 0000000000..d74652325a --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.notification.NotificationRequestDao; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.PartitionService; + +import java.util.List; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class NotificationsCleanUpServiceTest { + + @Mock + private PartitionService partitionService; + @Mock + private SqlPartitioningRepository partitioningRepository; + @Mock + private NotificationRequestDao notificationRequestDao; + @Mock + private TenantService tenantService; + + private NotificationsCleanUpService cleanUpService; + + private static final int BATCH_SIZE = 3; + + @BeforeEach + public void setUp() { + cleanUpService = new NotificationsCleanUpService(partitionService, partitioningRepository, notificationRequestDao, tenantService); + ReflectionTestUtils.setField(cleanUpService, "ttlInSec", 2592000L); + ReflectionTestUtils.setField(cleanUpService, "partitionSizeInHours", 168); + ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); + } + + @Test + public void testBatchLoopCallsDaoMultipleTimes() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(System.currentTimeMillis()); + + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); + + // Sysadmin: returns 3 (full batch), then 1 (partial) -> 2 calls + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(1); + // Tenant: returns 3, 3, 0 -> 3 calls + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(BATCH_SIZE) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao, times(2)) + .removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE)); + verify(notificationRequestDao, times(3)) + .removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); + } + + @Test + public void testSkipsTenantNotOnMyPartition() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); + when(partitionService.resolve(any(), eq(TenantId.SYS_TENANT_ID), eq(TenantId.SYS_TENANT_ID))) + .thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(System.currentTimeMillis()); + + // Sysadmin: no records + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); + TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); + when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); + when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); + + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao).removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); + verify(notificationRequestDao, never()).removeByTenantIdAndCreatedTimeBeforeBatch(eq(otherTenant), anyLong(), anyInt()); + } + + @Test + public void testNoPartitionsDropped_stillCleansUpRequests() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(0L); + + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(), 0, 0, false)); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao).removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE)); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java new file mode 100644 index 0000000000..22cc1be1dd --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.rpc; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.rpc.RpcDao; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.PartitionService; + +import java.util.List; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class RpcCleanUpServiceTest { + + @Mock + private PartitionService partitionService; + @Mock + private RpcDao rpcDao; + @Mock + private TenantService tenantService; + @Mock + private TbTenantProfileCache tenantProfileCache; + + private RpcCleanUpService cleanUpService; + + private static final int BATCH_SIZE = 3; + + @BeforeEach + public void setUp() { + cleanUpService = new RpcCleanUpService(tenantService, partitionService, tenantProfileCache, rpcDao); + ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); + } + + @Test + public void testBatchLoopCallsDaoMultipleTimes() { + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + setupTenant(tenantId, 7); + + // Returns 3 (full batch), 3 (full batch), 1 (partial) -> 3 calls + when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(BATCH_SIZE) + .thenReturn(1); + + cleanUpService.cleanUp(); + + verify(rpcDao, times(3)).deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); + } + + @Test + public void testSkipsTenantNotOnMyPartition() { + TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); + TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); + + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); + + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); + when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); + when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); + + setupTenantProfile(myTenant, 7); + when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(rpcDao).deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); + verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(eq(otherTenant), anyLong(), anyInt()); + } + + @Test + public void testSkipsTenantWithZeroTtl() { + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + setupTenant(tenantId, 0); + + cleanUpService.cleanUp(); + + verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(any(), anyLong(), anyInt()); + } + + private void setupTenant(TenantId tenantId, int rpcTtlDays) { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), eq(tenantId), eq(tenantId))).thenReturn(myPartition); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); + setupTenantProfile(tenantId, rpcTtlDays); + } + + private void setupTenantProfile(TenantId tenantId, int rpcTtlDays) { + TenantProfile profile = new TenantProfile(); + TenantProfileData profileData = new TenantProfileData(); + DefaultTenantProfileConfiguration config = new DefaultTenantProfileConfiguration(); + config.setRpcTtlDays(rpcTtlDays); + profileData.setConfiguration(config); + profile.setProfileData(profileData); + when(tenantProfileCache.get(tenantId)).thenReturn(profile); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java index 9bed9bc483..35e12691c9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java @@ -171,7 +171,14 @@ public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { if (this.leshanClient != null) { log.info("Stop/reboot LwM2M client {}", this.leshanClient.getEndpoint(identity)); - this.leshanClient.stop(false); + try { + this.leshanClient.stop(false); + } catch (Exception stopEx) { + // Leshan may throw NPE during CoAP observe-relation cleanup when the server + // reference is null (race condition in NotificationDataStore.toKey()). + // The client is still considered stopped at this point — proceed with restart. + log.warn("Exception during LwM2M client stop, proceeding with restart: {}", stopEx.getMessage()); + } log.info("Start after update fw LwM2M client {}", this.leshanClient.getEndpoint(identity)); this.leshanClient.start(); @@ -193,7 +200,7 @@ public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { } catch (Exception e) { log.error("Error during firmware update", e); } - }, 0, TimeUnit.SECONDS); // start immediately, without further delay + }, 1, TimeUnit.SECONDS); // delay 1 sec to allow CoAP Execute response to be delivered before client stops } protected void setLeshanClient(LeshanClient leshanClient) { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java index 432fac6d1a..c61ca0b123 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper; import org.thingsboard.server.transport.lwm2m.rpc.AbstractRpcLwM2MIntegrationTest; import java.util.concurrent.atomic.AtomicReference; import static java.util.concurrent.TimeUnit.SECONDS; @@ -37,6 +38,12 @@ import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID @Slf4j public class RpcLwm2mIntegrationReadCollectedValueTest extends AbstractRpcLwM2MIntegrationTest { + @Before + public void resetCollectedValueTimestamps() { + Lwm2mTestHelper.RESOURCE_ID_3303_12_5700_TS_0 = 0; + Lwm2mTestHelper.RESOURCE_ID_3303_12_5700_TS_1 = 0; + } + /** * Read {"id":"/3303/12/5700"} * Trigger a Send operation from the client with multiple values for the same resource as a payload diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index a7307b2308..5f43aa7e00 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -420,6 +420,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt Awaitility.await() .atMost(10, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() { }); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java index 4e4d0debf9..a9bdbb55db 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java @@ -191,6 +191,7 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte AtomicReference device = new AtomicReference<>(); await(alias + "find device [" + deviceName + "] after created") .atMost(200, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { device.set(doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class)); return device.get() != null; @@ -236,6 +237,7 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte AtomicReference device = new AtomicReference<>(); await(alias + "find device [" + deviceName + "] after created") .atMost(200, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { device.set(doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class)); return device.get() != null; diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java index 756c8e603c..d42adfbcf0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java @@ -468,6 +468,7 @@ public abstract class AbstractMqttV5ClientSparkplugAttributesTest extends Abstra AtomicReference> actualKeys = new AtomicReference<>(); await(alias + SparkplugMessageType.NBIRTH.name()) .atMost(40, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { actualKeys.set(doGetAsyncTyped(urlTemplate, new TypeReference<>() { })); @@ -483,6 +484,7 @@ public abstract class AbstractMqttV5ClientSparkplugAttributesTest extends Abstra AtomicReference> actualKeys = new AtomicReference<>(); await(alias + SparkplugMessageType.DBIRTH.name()) .atMost(40, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { actualKeys.set(doGetAsyncTyped(urlTemplate, new TypeReference<>() { })); diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index e79289340c..7f0ab964d6 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -44,6 +44,7 @@ queue.transport_api.response_poll_interval=5 queue.transport.poll_interval=5 queue.core.poll-interval=5 queue.core.partitions=2 +queue.core.housekeeper.task-reprocessing-delay-ms=0 queue.rule-engine.poll-interval=5 queue.rule-engine.stats.enabled=true diff --git a/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java b/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java index 798549cf85..cdb8c23c41 100644 --- a/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java +++ b/common/dao-api/src/main/java/org/thingsboard/server/dao/pat/ApiKeyService.java @@ -24,10 +24,14 @@ import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.pat.ApiKeyInfo; import org.thingsboard.server.dao.entity.EntityDaoService; +import java.util.List; + public interface ApiKeyService extends EntityDaoService { ApiKey saveApiKey(TenantId tenantId, ApiKeyInfo apiKey); + ApiKey saveApiKey(TenantId tenantId, ApiKeyInfo apiKeyInfo, String value, boolean doValidate); + void deleteApiKey(TenantId tenantId, ApiKey apiKey, boolean force); void deleteByUserId(TenantId tenantId, UserId userId); @@ -38,4 +42,8 @@ public interface ApiKeyService extends EntityDaoService { PageData findApiKeysByUserId(TenantId tenantId, UserId userId, PageLink pageLink); + List findApiKeysByUserId(TenantId tenantId, UserId userId); + + PageData findApiKeysByTenantId(TenantId tenantId, PageLink pageLink); + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java index 021639bc8c..2d3928ee88 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/cf/CalculatedField.java @@ -17,15 +17,15 @@ package org.thingsboard.server.common.data.cf; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.databind.JsonNode; import io.swagger.v3.oas.annotations.media.Schema; import jakarta.validation.Valid; import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.Setter; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityType; +import org.thingsboard.server.common.data.HasAdditionalInfo; import org.thingsboard.server.common.data.HasDebugSettings; import org.thingsboard.server.common.data.HasName; import org.thingsboard.server.common.data.HasTenantId; @@ -39,7 +39,6 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.validation.Length; import org.thingsboard.server.common.data.validation.NoXss; -import java.io.Serial; import java.util.Collections; import java.util.EnumSet; import java.util.Map; @@ -48,10 +47,7 @@ import java.util.Set; @Schema @Data @EqualsAndHashCode(callSuper = true) -public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, HasDebugSettings { - - @Serial - private static final long serialVersionUID = 4491966747773381420L; +public class CalculatedField extends BaseData implements HasName, HasTenantId, HasVersion, HasDebugSettings, HasAdditionalInfo { public static final Map> SUPPORTED_ENTITIES = Map.of( EntityType.DEVICE, CalculatedFieldType.all, @@ -90,28 +86,17 @@ public class CalculatedField extends BaseData implements HasN @Valid @NotNull private CalculatedFieldConfiguration configuration; - @Getter - @Setter private Long version; + @NoXss + @Schema(description = "Additional parameters of the calculated field") + private JsonNode additionalInfo; - public CalculatedField() { - super(); - } + public CalculatedField() {} public CalculatedField(CalculatedFieldId id) { super(id); } - public CalculatedField(TenantId tenantId, EntityId entityId, CalculatedFieldType type, String name, int configurationVersion, CalculatedFieldConfiguration configuration, Long version) { - this.tenantId = tenantId; - this.entityId = entityId; - this.type = type; - this.name = name; - this.configurationVersion = configurationVersion; - this.configuration = configuration; - this.version = version; - } - public CalculatedField(CalculatedField calculatedField) { super(calculatedField); this.tenantId = calculatedField.tenantId; @@ -123,6 +108,7 @@ public class CalculatedField extends BaseData implements HasN this.configurationVersion = calculatedField.configurationVersion; this.configuration = calculatedField.configuration; this.version = calculatedField.version; + this.additionalInfo = calculatedField.additionalInfo; } @Schema(description = "JSON object with the Calculated Field Id. Referencing non-existing Calculated Field Id will cause error.") @@ -159,6 +145,7 @@ public class CalculatedField extends BaseData implements HasN .append(", name='").append(name) .append(", configurationVersion=").append(configurationVersion) .append(", configuration=").append(configuration) + .append(", additionalInfo=").append(additionalInfo) .append(", version=").append(version) .append(", createdTime=").append(createdTime) .append(", id=").append(id).append(']') diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java index 834e19a1c2..474a0270c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/edge/EdgeEventType.java @@ -48,7 +48,8 @@ public enum EdgeEventType { OAUTH2_CLIENT(true, EntityType.OAUTH2_CLIENT), DOMAIN(true, EntityType.DOMAIN), CALCULATED_FIELD(false, EntityType.CALCULATED_FIELD), - AI_MODEL(true, EntityType.AI_MODEL); + AI_MODEL(true, EntityType.AI_MODEL), + API_KEY(true, EntityType.API_KEY); private final boolean allEdgesRelated; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java index dea590295e..d66ec046de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java @@ -23,6 +23,7 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -32,6 +33,9 @@ import java.util.UUID; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9214680001573764374L; + private List alarms; public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java index 0313190056..445850d387 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -33,6 +34,9 @@ import java.util.UUID; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9156667024462937756L; + private String userTitle; private List alarms; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index fe25a98a1d..c1a233b1c4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -23,6 +23,7 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -32,6 +33,9 @@ import java.util.UUID; @NoArgsConstructor public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9009068831061529286L; + private EntityType entityType; private List entities; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 875ef2765f..2df7cf4dd4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.io.Serializable; @JsonIgnoreProperties(ignoreUnknown = true) @@ -45,6 +46,9 @@ import java.io.Serializable; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class HousekeeperTask implements Serializable { + @Serial + private static final long serialVersionUID = -2585974110832225152L; + private TenantId tenantId; private EntityId entityId; private HousekeeperTaskType taskType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java index cd3e94e5c6..931c2931cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class LatestTsDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 5193191938513490138L; + private String key; public LatestTsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java index 443d929d8b..be7ff6f7ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor public class TenantEntitiesDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = -8033108795318393447L; + private EntityType entityType; public TenantEntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java index d9315f0ff4..b520899ca4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 4573851542705079043L; + private String key; public TsHistoryDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java index 7baa8e72f2..ae033f769c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/id/EntityIdFactory.java @@ -114,6 +114,7 @@ public class EntityIdFactory { case DOMAIN -> new DomainId(uuid); case CALCULATED_FIELD -> new CalculatedFieldId(uuid); case AI_MODEL -> new AiModelId(uuid); + case API_KEY -> new ApiKeyId(uuid); case ADMIN_SETTINGS -> new AdminSettingsId(uuid); default -> throw new IllegalArgumentException("EdgeEventType " + edgeEventType + " is not supported!"); }; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index cb4092f433..595e1aa26b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -37,4 +37,9 @@ public interface TsKvEntry extends KvEntry, HasVersion { return new TsValue(getTs(), getValueAsString()); } + @JsonIgnore + default boolean isDeletedEntry() { + return getTs() == 0 && (getValue() == null || getValueAsString().isEmpty()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java index 0b383a6d22..0475a9a02d 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java @@ -16,6 +16,7 @@ package org.thingsboard.server.common.data.notification.rule; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -46,6 +47,7 @@ import java.util.UUID; @DiscriminatorMapping(value = "TASK_PROCESSING_FAILURE", schema = DefaultNotificationRuleRecipientsConfig.TaskProcessingFailureRecipientsConfig.class), @DiscriminatorMapping(value = "RESOURCES_SHORTAGE", schema = DefaultNotificationRuleRecipientsConfig.ResourceShortageRecipientsConfig.class) }) +@JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "triggerType", include = JsonTypeInfo.As.EXISTING_PROPERTY) @JsonSubTypes({ @Type(name = "ALARM", value = EscalatedNotificationRuleRecipientsConfig.class), diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 472ce8e8dd..48ec26f875 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -47,8 +47,10 @@ enum EdgeVersion { V_4_3_0 = 13; V_4_2_1_2 = 14; V_4_2_2 = 4220; + V_4_2_2_1 = 4221; V_4_3_0_1 = 15; V_4_3_1 = 4310; + V_4_3_1_1 = 4311; V_4_4_0 = 4400; V_LATEST = 99999; @@ -146,6 +148,13 @@ message AiModelUpdateMsg{ string entity = 4; } +message ApiKeyUpdateMsg{ + UpdateMsgType msgType = 1; + int64 idMSB = 2; + int64 idLSB = 3; + string entity = 4; +} + message EntityDataProto { int64 entityIdMSB = 1; int64 entityIdLSB = 2; @@ -456,6 +465,7 @@ message UplinkMsg { repeated AiModelUpdateMsg aiModelUpdateMsg = 27; repeated UserUpdateMsg userUpdateMsg = 28; repeated UserCredentialsUpdateMsg userCredentialsUpdateMsg = 29; + repeated ApiKeyUpdateMsg apiKeyUpdateMsg = 30; } message UplinkResponseMsg { @@ -507,4 +517,5 @@ message DownlinkMsg { repeated OAuth2DomainUpdateMsg oAuth2DomainUpdateMsg = 34; repeated CalculatedFieldUpdateMsg calculatedFieldUpdateMsg = 35; repeated AiModelUpdateMsg aiModelUpdateMsg = 36; + repeated ApiKeyUpdateMsg apiKeyUpdateMsg = 37; } diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java index 4b2ef07994..8158e82a81 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java @@ -323,22 +323,24 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab @Override public Iterator getAllRegistrations() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { Collection list = new LinkedList<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(REG_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); - list.add(deserializeReg(element)); + byte[] element = getConnection.get(key); + if (element != null) { + list.add(deserializeReg(element)); + } }); }); return list.iterator(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java index 3293cd8b53..4beefb4896 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java @@ -61,21 +61,21 @@ public class TbRedisLwM2MClientStore implements TbLwM2MClientStore { @Override public Set getAll() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { Set clients = new HashSet<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(CLIENT_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); + byte[] element = getConnection.get(key); if (element != null) { try { clients.add(deserialize(element)); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java index 73b6f3c8df..31a78234d0 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java @@ -35,22 +35,24 @@ public class TbRedisLwM2MModelConfigStore implements TbLwM2MModelConfigStore { @Override public List getAll() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { List configs = new ArrayList<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(MODEL_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); - configs.add(JacksonUtil.fromBytes(element, LwM2MModelConfig.class)); + byte[] element = getConnection.get(key); + if (element != null) { + configs.add(JacksonUtil.fromBytes(element, LwM2MModelConfig.class)); + } }); }); return configs; diff --git a/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java b/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java new file mode 100644 index 0000000000..9fe29c1188 --- /dev/null +++ b/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.store; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.ScanOptions; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2MClientState; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; + +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.thingsboard.server.transport.lwm2m.server.store.util.LwM2MClientSerDes.serialize; + +/** + * Verifies that {@link TbRedisLwM2MClientStore#getAll()} uses separate connections for + * SCAN and GET operations to prevent Jedis 5.x response-ordering corruption that occurs + * when both commands share the same connection. + */ +@ExtendWith(MockitoExtension.class) +class TbRedisLwM2MClientStoreTest { + + @Mock + RedisConnectionFactory connectionFactory; + + @Mock + RedisConnection scanConnection; + + @Mock + RedisConnection getConnection; + + TbRedisLwM2MClientStore store; + + @BeforeEach + void setUp() { + // First getConnection() call → scanConnection, second → getConnection + when(connectionFactory.getConnection()) + .thenReturn(scanConnection) + .thenReturn(getConnection); + store = new TbRedisLwM2MClientStore(connectionFactory); + } + + @Test + void getAll_returnsSingleClient() { + LwM2mClient client = new LwM2mClient("nodeId", "testEndpoint"); + client.setState(LwM2MClientState.REGISTERED); + byte[] key = "CLIENT#EP#testEndpoint".getBytes(); + byte[] value = serialize(client); + + // Cursor created before thenReturn to avoid Mockito unfinished-stubbing error + Cursor cursor = cursorOf(key); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + when(getConnection.get(key)).thenReturn(value); + + Set result = store.getAll(); + + assertThat(result).hasSize(1); + assertThat(result.iterator().next().getEndpoint()).isEqualTo("testEndpoint"); + } + + @Test + void getAll_getIsNeverCalledOnScanConnection() { + Cursor cursor = cursorOf(); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + + store.getAll(); + + verify(scanConnection, never()).get(any(byte[].class)); + } + + @Test + void getAll_scanIsNeverCalledOnGetConnection() { + Cursor cursor = cursorOf(); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + + store.getAll(); + + verify(getConnection, never()).scan(any(ScanOptions.class)); + } + + @Test + void getAll_skipsKeyWhenValueIsNull() { + byte[] key = "CLIENT#EP#gone".getBytes(); + Cursor cursor = cursorOf(key); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + // getConnection.get(key) returns null by default — no stubbing needed + + Set result = store.getAll(); + + assertThat(result).isEmpty(); + } + + /** + * Creates a mock {@link Cursor} that iterates over the given keys via {@code forEachRemaining}. + * The cursor is created separately (not inside a {@code thenReturn()} argument) to avoid + * Mockito's "unfinished stubbing" error caused by nested {@code when()} calls. + */ + @SuppressWarnings("unchecked") + private static Cursor cursorOf(byte[]... keys) { + Cursor cursor = mock(Cursor.class); + List keyList = List.of(keys); + doAnswer(inv -> { + Consumer action = inv.getArgument(0); + keyList.forEach(action); + return null; + }).when(cursor).forEachRemaining(any(Consumer.class)); + return cursor; + } +} diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java index 3d05e999e0..10245c5a24 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportContext.java @@ -88,20 +88,30 @@ public class MqttTransportContext extends TransportContext { @Value("${transport.mqtt.proxy_enabled:false}") private boolean proxyEnabled; - private final AtomicInteger connectionsCounter = new AtomicInteger(); + private final AtomicInteger connectionsActiveCounterMQTT = new AtomicInteger(); + private final AtomicInteger connectionsActiveCounterMQTTS = new AtomicInteger(); @PostConstruct public void init() { super.init(); - transportService.createGaugeStats("openConnections", connectionsCounter); + transportService.createGaugeStats("connections_active", connectionsActiveCounterMQTT, "protocol", "MQTT"); + transportService.createGaugeStats("connections_active", connectionsActiveCounterMQTTS, "protocol", "MQTTS"); } - public void channelRegistered() { - connectionsCounter.incrementAndGet(); + public void channelRegistered(boolean isSSL) { + if (isSSL) { + connectionsActiveCounterMQTTS.incrementAndGet(); + } else { + connectionsActiveCounterMQTT.incrementAndGet(); + } } - public void channelUnregistered() { - connectionsCounter.decrementAndGet(); + public void channelUnregistered(boolean isSSL) { + if (isSSL) { + connectionsActiveCounterMQTTS.decrementAndGet(); + } else { + connectionsActiveCounterMQTT.decrementAndGet(); + } } public boolean checkAddress(InetSocketAddress address) { diff --git a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java index f753579292..48fbad3032 100644 --- a/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java +++ b/common/transport/mqtt/src/main/java/org/thingsboard/server/transport/mqtt/MqttTransportHandler.java @@ -180,16 +180,20 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement this.rpcAwaitingAck = new ConcurrentHashMap<>(); } + boolean isSSL() { + return sslHandler != null; + } + @Override public void channelRegistered(ChannelHandlerContext ctx) throws Exception { super.channelRegistered(ctx); - context.channelRegistered(); + context.channelRegistered(isSSL()); } @Override public void channelUnregistered(ChannelHandlerContext ctx) throws Exception { super.channelUnregistered(ctx); - context.channelUnregistered(); + context.channelUnregistered(isSSL()); } @Override @@ -254,6 +258,17 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement } } + String getClientAddr(ChannelHandlerContext ctx) { + try { + InetSocketAddress remote = getAddress(ctx); + if (remote == null) return "unknown"; + String host = remote.getAddress() != null ? remote.getAddress().getHostAddress() : remote.getHostString(); + return host + ":" + remote.getPort(); + } catch (Exception ignored) { + return "unknown"; + } + } + InetSocketAddress getAddress(ChannelHandlerContext ctx) { var address = ctx.channel().attr(MqttTransportService.ADDRESS).get(); if (address == null) { @@ -1150,19 +1165,25 @@ public class MqttTransportHandler extends ChannelInboundHandlerAdapter implement public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { if (cause instanceof IOException) { if (log.isDebugEnabled()) { - log.debug("[{}][{}][{}] IOException: {}", sessionId, + String clientAddr = getClientAddr(ctx); + log.debug("[{}][{}][{}][{}] {}: {}", sessionId, Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null), Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""), + clientAddr, + cause.getClass().getSimpleName(), cause.getMessage(), cause); } else if (log.isInfoEnabled()) { - log.info("[{}][{}][{}] IOException: {}", sessionId, + String clientAddr = getClientAddr(ctx); + log.info("[{}][{}][{}][{}] {}: {}", sessionId, Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceId).orElse(null), Optional.ofNullable(this.deviceSessionCtx.getDeviceInfo()).map(TransportDeviceInfo::getDeviceName).orElse(""), + clientAddr, + cause.getClass().getSimpleName(), cause.getMessage()); } } else { - log.error("[{}] Unexpected Exception", sessionId, cause); + log.error("[{}][{}] Unexpected Exception", sessionId, getClientAddr(ctx), cause); } closeCtx(ctx, MqttReasonCodes.Disconnect.SERVER_SHUTTING_DOWN); diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java index 5c7d552855..a8f8a8b863 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/TransportService.java @@ -161,5 +161,5 @@ public interface TransportService { boolean hasSession(SessionInfoProto sessionInfo); - void createGaugeStats(String openConnections, AtomicInteger connectionsCounter); + void createGaugeStats(String statsName, AtomicInteger number, String... tags); } diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java index 260957c990..e8ade498db 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/service/DefaultTransportService.java @@ -1257,9 +1257,21 @@ public class DefaultTransportService extends TransportActivityManager implements } @Override - public void createGaugeStats(String statsName, AtomicInteger number) { - statsFactory.createGauge(StatsType.TRANSPORT + "." + statsName, number); - statsMap.put(statsName, number); + public void createGaugeStats(String statsName, AtomicInteger number, String... tags) { + String key = "thingsboard" + "." + StatsType.TRANSPORT.getName() + "." + statsName; + statsFactory.createGauge(key, number, tags); + statsMap.put(statsName + tagsKey(tags), number); + } + + private String tagsKey(String... tags) { + if (tags == null || tags.length < 2) return ""; + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < tags.length; i += 2) { + if (i > 0) sb.append(','); + sb.append(tags[i]).append('=').append(i + 1 < tags.length ? tags[i + 1] : ""); + } + sb.append(']'); + return sb.toString(); } @Scheduled(fixedDelayString = "${transport.stats.print-interval-ms:60000}") diff --git a/dao/pom.xml b/dao/pom.xml index 928a107079..d526eeed4f 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -280,7 +280,6 @@ org.apache.maven.plugins maven-jar-plugin - ${jar-plugin.version} diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java index 9f9e4a67e9..69a04f301b 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/ModelConstants.java @@ -730,6 +730,7 @@ public class ModelConstants { public static final String CALCULATED_FIELD_CONFIGURATION_VERSION = "configuration_version"; public static final String CALCULATED_FIELD_CONFIGURATION = "configuration"; public static final String CALCULATED_FIELD_VERSION = "version"; + public static final String CALCULATED_FIELD_ADDITIONAL_INFO = ADDITIONAL_INFO_PROPERTY; /** * Tasks constants. diff --git a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java index 5a507b7fd3..47c1520ac0 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java +++ b/dao/src/main/java/org/thingsboard/server/dao/model/sql/CalculatedFieldEntity.java @@ -36,6 +36,7 @@ import org.thingsboard.server.dao.util.mapping.JsonConverter; import java.util.UUID; +import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ADDITIONAL_INFO; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_CONFIGURATION_VERSION; import static org.thingsboard.server.dao.model.ModelConstants.CALCULATED_FIELD_ENTITY_ID; @@ -81,9 +82,11 @@ public class CalculatedFieldEntity extends BaseVersionedEntity @Column(name = DEBUG_SETTINGS) private String debugSettings; - public CalculatedFieldEntity() { - super(); - } + @Convert(converter = JsonConverter.class) + @Column(name = CALCULATED_FIELD_ADDITIONAL_INFO) + private JsonNode additionalInfo; + + public CalculatedFieldEntity() {} public CalculatedFieldEntity(CalculatedField calculatedField) { this.setUuid(calculatedField.getUuidId()); @@ -97,6 +100,7 @@ public class CalculatedFieldEntity extends BaseVersionedEntity this.configuration = JacksonUtil.valueToTree(calculatedField.getConfiguration()); this.version = calculatedField.getVersion(); this.debugSettings = JacksonUtil.toString(calculatedField.getDebugSettings()); + this.additionalInfo = calculatedField.getAdditionalInfo(); } @Override @@ -111,6 +115,7 @@ public class CalculatedFieldEntity extends BaseVersionedEntity calculatedField.setConfiguration(JacksonUtil.treeToValue(configuration, CalculatedFieldConfiguration.class)); calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); + calculatedField.setAdditionalInfo(additionalInfo); return calculatedField; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java index 96a86073d4..8e1408f4fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java @@ -50,7 +50,7 @@ public interface NotificationRequestDao extends Dao { boolean existsByTenantIdAndStatusAndTemplateId(TenantId tenantId, NotificationRequestStatus status, NotificationTemplateId templateId); - int removeAllByCreatedTimeBefore(long ts); + int removeByTenantIdAndCreatedTimeBeforeBatch(TenantId tenantId, long ts, int batchSize); NotificationRequestInfo findInfoById(TenantId tenantId, NotificationRequestId id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 484b3c49b3..2e30c0d598 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -196,7 +196,9 @@ public class BaseOtaPackageService extends AbstractCachedEntityService INCORRECT_OTA_PACKAGE_ID + id); try { + Long oid = getDataOidById(tenantId, otaPackageId); otaPackageDao.removeById(tenantId, otaPackageId.getId()); + unlinkDataIfPresent(tenantId, otaPackageId, oid); publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(otaPackageId).build()); } catch (Exception t) { @@ -215,6 +217,30 @@ public class BaseOtaPackageService extends AbstractCachedEntityService tenantOtaPackageRemover = - new PaginatedRemover<>() { - - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); - } + private final PaginatedRemover tenantOtaPackageRemover = new PaginatedRemover<>() { + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { - deleteOtaPackage(tenantId, entity.getId()); - } - }; + @Override + protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { + deleteOtaPackage(tenantId, entity.getId()); + } + }; @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index c11a13cbe1..875aaea4ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -18,15 +18,20 @@ package org.thingsboard.server.dao.ota; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; +import java.util.UUID; + public interface OtaPackageDao extends Dao, TenantEntityWithDataDao, ExportableEntityDao { Long sumDataSizeByTenantId(TenantId tenantId); OtaPackage findOtaPackageByTenantIdAndTitleAndVersion(TenantId tenantId, String title, String version); + Long getDataOidById(UUID id); + + Integer unlinkLargeObject(Long dataOid); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java index d71d6502cf..f5ff79fb52 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyDao.java @@ -17,15 +17,22 @@ package org.thingsboard.server.dao.pat; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.dao.Dao; +import java.util.List; import java.util.Set; public interface ApiKeyDao extends Dao { ApiKey findByValue(String value); + PageData findByTenantId(TenantId tenantId, PageLink pageLink); + + List findByTenantIdAndUserId(TenantId tenantId, UserId userId); + Set deleteByTenantId(TenantId tenantId); Set deleteByUserId(TenantId tenantId, UserId userId); diff --git a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java index 17a3e2bda3..36d8736f78 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/pat/ApiKeyServiceImpl.java @@ -24,6 +24,7 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.event.TransactionalEventListener; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.exception.DataValidationException; import org.thingsboard.server.common.data.id.ApiKeyId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -34,9 +35,11 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.common.data.pat.ApiKeyInfo; import org.thingsboard.server.dao.entity.AbstractCachedEntityService; +import org.thingsboard.server.dao.eventsourcing.DeleteEntityEvent; import org.thingsboard.server.dao.eventsourcing.SaveEntityEvent; import org.thingsboard.server.dao.service.validator.ApiKeyDataValidator; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -73,13 +76,20 @@ public class ApiKeyServiceImpl extends AbstractCachedEntityService findApiKeysByUserId(TenantId tenantId, UserId userId) { + log.trace("Executing findApiKeysByUserId [{}][{}]", tenantId, userId); + validateId(userId, id -> INCORRECT_USER_ID + id); + return apiKeyDao.findByTenantIdAndUserId(tenantId, userId); + } + @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { return Optional.ofNullable(findApiKeyById(tenantId, new ApiKeyId(entityId.getId()))); @@ -126,6 +143,20 @@ public class ApiKeyServiceImpl extends AbstractCachedEntityService INCORRECT_API_KEY_ID + id); apiKeyDao.removeById(tenantId, apiKeyId); publishEvictEvent(new ApiKeyEvictEvent(apiKey.getValue())); + eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(apiKey.getId()).build()); + } + + @Override + public void deleteEntity(TenantId tenantId, EntityId id, boolean force) { + ApiKey apiKey = findApiKeyById(tenantId, new ApiKeyId(id.getId())); + if (apiKey == null) { + if (force) { + return; + } else { + throw new DataValidationException("Unable to delete non-existent API key."); + } + } + deleteApiKey(tenantId, apiKey, force); } @Override @@ -144,6 +175,13 @@ public class ApiKeyServiceImpl extends AbstractCachedEntityService publishEvictEvent(new ApiKeyEvictEvent(value))); } + @Override + public PageData findApiKeysByTenantId(TenantId tenantId, PageLink pageLink) { + log.trace("Executing findApiKeysByTenantId [{}]", tenantId); + validateId(tenantId, id -> INCORRECT_TENANT_ID + id); + return apiKeyDao.findByTenantId(tenantId, pageLink); + } + @Override public ApiKey findApiKeyByValue(String value) { log.trace("Executing findApiKeyByValue [{}]", value); diff --git a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java index f88b672a34..37fe2950b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java @@ -24,12 +24,13 @@ import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.Dao; public interface RpcDao extends Dao { + PageData findAllByDeviceId(TenantId tenantId, DeviceId deviceId, PageLink pageLink); PageData findAllByDeviceIdAndStatus(TenantId tenantId, DeviceId deviceId, RpcStatus rpcStatus, PageLink pageLink); PageData findAllRpcByTenantId(TenantId tenantId, PageLink pageLink); - int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime); + int deleteOutdatedRpcByTenantIdBatch(TenantId tenantId, Long expirationTime, int batchSize); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java index f6113155ca..0f9dcc20be 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/cf/DefaultNativeCalculatedFieldRepository.java @@ -74,6 +74,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF JsonNode configuration = JacksonUtil.toJsonNode((String) row.get("configuration")); long version = row.get("version") != null ? (long) row.get("version") : 0; String debugSettings = (String) row.get("debug_settings"); + JsonNode additionalInfo = JacksonUtil.toJsonNode((String) row.get("additional_info")); CalculatedField calculatedField = new CalculatedField(); calculatedField.setId(new CalculatedFieldId(id)); @@ -91,6 +92,7 @@ public class DefaultNativeCalculatedFieldRepository implements NativeCalculatedF } calculatedField.setVersion(version); calculatedField.setDebugSettings(JacksonUtil.fromString(debugSettings, DebugSettings.class)); + calculatedField.setAdditionalInfo(additionalInfo); return calculatedField; }).collect(Collectors.toList()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java index 9d32e91ca2..f4cc406a75 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java @@ -98,8 +98,8 @@ public class JpaNotificationRequestDao extends JpaAbstractDao findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + // The 'data' column is of type OID (PostgreSQL large object reference), so it returns the OID as Long + @Query(value = "SELECT data FROM ota_package WHERE id = :id AND data IS NOT NULL", nativeQuery = true) + Long getDataOidById(@Param("id") UUID id); + + @Transactional + @Query(value = "SELECT lo_unlink(:oid)", nativeQuery = true) + Integer unlinkLargeObject(@Param("oid") Long oid); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java index e3e560725c..a83223d726 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/ApiKeyRepository.java @@ -15,6 +15,8 @@ */ package org.thingsboard.server.dao.sql.pat; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; @@ -22,6 +24,7 @@ import org.springframework.data.repository.query.Param; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.dao.model.sql.ApiKeyEntity; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -29,6 +32,10 @@ public interface ApiKeyRepository extends JpaRepository { ApiKeyEntity findByValue(String value); + Page findByTenantId(UUID tenantId, Pageable pageable); + + List findByTenantIdAndUserId(UUID tenantId, UUID userId); + @Transactional @Modifying @Query(value = """ diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java index 61ca7bd142..69de36384e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/pat/JpaApiKeyDao.java @@ -22,6 +22,8 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.pat.ApiKey; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.ApiKeyEntity; @@ -29,6 +31,7 @@ import org.thingsboard.server.dao.pat.ApiKeyDao; import org.thingsboard.server.dao.sql.JpaAbstractDao; import org.thingsboard.server.dao.util.SqlDao; +import java.util.List; import java.util.Set; import java.util.UUID; @@ -45,6 +48,16 @@ public class JpaApiKeyDao extends JpaAbstractDao implement return DaoUtil.getData(apiKeyRepository.findByValue(value)); } + @Override + public PageData findByTenantId(TenantId tenantId, PageLink pageLink) { + return DaoUtil.toPageData(apiKeyRepository.findByTenantId(tenantId.getId(), DaoUtil.toPageable(pageLink))); + } + + @Override + public List findByTenantIdAndUserId(TenantId tenantId, UserId userId) { + return DaoUtil.convertDataList(apiKeyRepository.findByTenantIdAndUserId(tenantId.getId(), userId.getId())); + } + @Override public Set deleteByTenantId(TenantId tenantId) { return apiKeyRepository.deleteByTenantId(tenantId.getId()); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java index 9daae69d2a..a929b6d1e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java @@ -71,8 +71,8 @@ public class JpaRpcDao extends JpaAbstractDao implements RpcDao, @Transactional @Override - public int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { - return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); + public int deleteOutdatedRpcByTenantIdBatch(TenantId tenantId, Long expirationTime, int batchSize) { + return rpcRepository.deleteOutdatedRpcByTenantIdBatch(tenantId.getId(), expirationTime, batchSize); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java index 8a9489333f..3f76170c84 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java @@ -21,20 +21,27 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.model.sql.RpcEntity; import java.util.UUID; public interface RpcRepository extends JpaRepository { + Page findAllByTenantIdAndDeviceId(UUID tenantId, UUID deviceId, Pageable pageable); Page findAllByTenantIdAndDeviceIdAndStatus(UUID tenantId, UUID deviceId, RpcStatus status, Pageable pageable); Page findAllByTenantId(UUID tenantId, Pageable pageable); + @Transactional @Modifying - @Query(value = "DELETE FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime", + @Query(value = "DELETE FROM rpc WHERE id IN " + + "(SELECT id FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime LIMIT :batchSize)", nativeQuery = true) - int deleteOutdatedRpcByTenantId(@Param("tenantId") UUID tenantId, @Param("expirationTime") Long expirationTime); + int deleteOutdatedRpcByTenantIdBatch(@Param("tenantId") UUID tenantId, + @Param("expirationTime") Long expirationTime, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/main/resources/sql/schema-entities.sql b/dao/src/main/resources/sql/schema-entities.sql index 3d9c9f17e6..d5e359b241 100644 --- a/dao/src/main/resources/sql/schema-entities.sql +++ b/dao/src/main/resources/sql/schema-entities.sql @@ -935,6 +935,7 @@ CREATE TABLE IF NOT EXISTS calculated_field ( configuration varchar(1000000), version BIGINT DEFAULT 1, debug_settings varchar(1024), + additional_info varchar, CONSTRAINT calculated_field_unq_key UNIQUE (entity_id, type, name) ); diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java index c9524ba7b5..3caf4bb5f4 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/CalculatedFieldServiceTest.java @@ -18,6 +18,7 @@ package org.thingsboard.server.dao.service; import org.apache.commons.lang3.RandomUtils; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.TenantProfile; @@ -83,6 +84,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { assertThat(savedCalculatedField.getType()).isEqualTo(calculatedField.getType()); assertThat(savedCalculatedField.getName()).isEqualTo(calculatedField.getName()); assertThat(savedCalculatedField.getConfiguration()).isEqualTo(calculatedField.getConfiguration()); + assertThat(savedCalculatedField.getAdditionalInfo()).isEqualTo(calculatedField.getAdditionalInfo()); assertThat(savedCalculatedField.getVersion()).isEqualTo(1L); savedCalculatedField.setName("Test CF"); @@ -90,6 +92,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { CalculatedField updatedCalculatedField = calculatedFieldService.save(savedCalculatedField); assertThat(updatedCalculatedField.getName()).isEqualTo(savedCalculatedField.getName()); + assertThat(updatedCalculatedField.getAdditionalInfo()).isEqualTo(savedCalculatedField.getAdditionalInfo()); assertThat(updatedCalculatedField.getVersion()).isEqualTo(savedCalculatedField.getVersion() + 1); } @@ -495,6 +498,7 @@ public class CalculatedFieldServiceTest extends AbstractServiceTest { calculatedField.setName("Test Calculated Field"); calculatedField.setConfigurationVersion(1); calculatedField.setConfiguration(getCalculatedFieldConfig(referencedEntityId)); + calculatedField.setAdditionalInfo(JacksonUtil.newObjectNode()); return calculatedField; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java index 002a5a7845..69fa3fbc5d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java @@ -36,14 +36,14 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; -import org.thingsboard.server.exception.DataValidationException; +import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; +import org.thingsboard.server.exception.DataValidationException; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -77,6 +77,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + OtaPackageDao otaPackageDao; + @Autowired TbTenantProfileCache tenantProfileCache; @Before @@ -118,10 +120,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); int maxSumDataSize = 8; - List packages = new ArrayList<>(maxSumDataSize); - for (int i = 2; i <= maxSumDataSize; i++) { - packages.add(createAndSaveFirmware(tenantId, "0." + i)); + createAndSaveFirmware(tenantId, "0." + i); Assert.assertEquals(i, otaPackageService.sumDataSizeByTenantId(tenantId)); } @@ -533,6 +533,39 @@ public class OtaPackageServiceTest extends AbstractServiceTest { Assert.assertNull(foundFirmware); } + @Test + public void testDeleteOtaPackageWithoutData() { + OtaPackageInfo firmwareInfo = new OtaPackageInfo(); + firmwareInfo.setTenantId(tenantId); + firmwareInfo.setDeviceProfileId(deviceProfileId); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle(TITLE); + firmwareInfo.setVersion(VERSION); + OtaPackageInfo savedFirmwareInfo = otaPackageService.saveOtaPackageInfo(firmwareInfo, false); + + Assert.assertNotNull(savedFirmwareInfo); + Assert.assertNotNull(savedFirmwareInfo.getId()); + + // Should not throw NPE when deleting package without data (OID is null) + otaPackageService.deleteOtaPackage(tenantId, savedFirmwareInfo.getId()); + + OtaPackageInfo foundFirmware = otaPackageService.findOtaPackageInfoById(tenantId, savedFirmwareInfo.getId()); + Assert.assertNull(foundFirmware); + } + + @Test + public void testDeleteOtaPackageUnlinksLargeObject() { + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); + + Long oid = otaPackageDao.getDataOidById(savedFirmware.getId().getId()); + Assert.assertNotNull(oid); + + otaPackageService.deleteOtaPackage(tenantId, savedFirmware.getId()); + + // Verify the large object was unlinked - PostgreSQL throws an exception when the object doesn't exist + assertThatThrownBy(() -> otaPackageDao.unlinkLargeObject(oid)).hasMessageContaining("large object " + oid + " does not exist"); + } + @Test public void testFindTenantFirmwaresByTenantId() { List firmwares = new ArrayList<>(); @@ -567,8 +600,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { } } while (pageData.hasNext()); - Collections.sort(firmwares, idComparator); - Collections.sort(loadedFirmwares, idComparator); + firmwares.sort(idComparator); + loadedFirmwares.sort(idComparator); assertThat(firmwares).isEqualTo(loadedFirmwares); @@ -622,8 +655,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { } } while (pageData.hasNext()); - Collections.sort(firmwares, idComparator); - Collections.sort(loadedFirmwares, idComparator); + firmwares.sort(idComparator); + loadedFirmwares.sort(idComparator); assertThat(firmwares).isEqualTo(loadedFirmwares); @@ -726,4 +759,5 @@ public class OtaPackageServiceTest extends AbstractServiceTest { firmware.setDataSize(DATA_SIZE); return firmware; } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java index aa43826734..b66cdb4363 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java @@ -65,7 +65,7 @@ public class TimeseriesServiceNoSqlTest extends BaseTimeseriesServiceTest { new BasicTsKvEntry(TimeUnit.MINUTES.toMillis(5), new JsonDataEntry("test", "{\"test\":\"testValue\"}"))); DeviceId deviceId = new DeviceId(Uuids.timeBased()); - tsService.save(tenantId, deviceId, timeseries, ttlInSec); + tsService.save(tenantId, deviceId, timeseries, ttlInSec).get(MAX_TIMEOUT, TimeUnit.SECONDS); List fullList = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery("test", 0L, TimeUnit.MINUTES.toMillis(6), 1000, 10, Aggregation.NONE))).get(MAX_TIMEOUT, TimeUnit.SECONDS); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java new file mode 100644 index 0000000000..5e2fe2308b --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.notification; + +import org.junit.After; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.id.NotificationRequestId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.notification.NotificationRequest; +import org.thingsboard.server.common.data.notification.NotificationRequestStatus; +import org.thingsboard.server.dao.AbstractJpaDaoTest; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JpaNotificationRequestDaoTest extends AbstractJpaDaoTest { + + @Autowired + JpaNotificationRequestDao notificationRequestDao; + + private final List createdRequests = new ArrayList<>(); + + @After + public void tearDown() { + for (NotificationRequest request : createdRequests) { + notificationRequestDao.removeById(request.getTenantId(), request.getId().getId()); + } + createdRequests.clear(); + } + + @Test + public void testBatchDeletion() { + TenantId sysTenantId = TenantId.SYS_TENANT_ID; + long now = System.currentTimeMillis(); + long oldTimestamp = now - TimeUnit.DAYS.toMillis(30); + + NotificationRequest oldRequest1 = createNotificationRequest(sysTenantId, oldTimestamp); + notificationRequestDao.save(sysTenantId, oldRequest1); + + NotificationRequest oldRequest2 = createNotificationRequest(sysTenantId, oldTimestamp); + notificationRequestDao.save(sysTenantId, oldRequest2); + + NotificationRequest freshRequest = createNotificationRequest(sysTenantId, now); + notificationRequestDao.save(sysTenantId, freshRequest); + + TenantId tenant2Id = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); + NotificationRequest tenant2Request = createNotificationRequest(tenant2Id, oldTimestamp); + notificationRequestDao.save(tenant2Id, tenant2Request); + + int batchSize = 10_000; + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, oldTimestamp - 1, batchSize)).isEqualTo(0); + + long expirationTime = now - TimeUnit.DAYS.toMillis(15); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, expirationTime, batchSize)).isEqualTo(2); + + assertThat(notificationRequestDao.findById(sysTenantId, freshRequest.getId().getId())).isNotNull(); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2Id, now + 1, batchSize)).isEqualTo(1); + } + + @Test + public void testBatchDeletionWithSmallBatchSize() { + TenantId tenantId = TenantId.SYS_TENANT_ID; + long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); + + for (int i = 0; i < 10; i++) { + NotificationRequest request = createNotificationRequest(tenantId, oldTimestamp); + notificationRequestDao.save(tenantId, request); + } + + int batchSize = 3; + long expirationTime = System.currentTimeMillis(); + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(1); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(0); + } + + @Test + public void testBatchDeletionIsolationBetweenTenants() { + TenantId tenant1 = TenantId.SYS_TENANT_ID; + TenantId tenant2 = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); + long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); + + for (int i = 0; i < 5; i++) { + NotificationRequest request = createNotificationRequest(tenant1, oldTimestamp); + notificationRequestDao.save(tenant1, request); + } + + for (int i = 0; i < 3; i++) { + NotificationRequest request = createNotificationRequest(tenant2, oldTimestamp); + notificationRequestDao.save(tenant2, request); + } + + int batchSize = 10_000; + long expirationTime = System.currentTimeMillis(); + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant1, expirationTime, batchSize)).isEqualTo(5); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2, expirationTime, batchSize)).isEqualTo(3); + } + + private NotificationRequest createNotificationRequest(TenantId tenantId, long createdTime) { + NotificationRequest request = new NotificationRequest(); + request.setId(new NotificationRequestId(UUID.randomUUID())); + request.setTenantId(tenantId); + request.setCreatedTime(createdTime); + request.setTargets(List.of(UUID.randomUUID())); + request.setStatus(NotificationRequestStatus.SENT); + createdRequests.add(request); + return request; + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java index 1629922685..921339a92b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java @@ -51,9 +51,10 @@ public class JpaRpcDaoTest extends AbstractJpaDaoTest { rpc.setDeviceId(new DeviceId(UUID.randomUUID())); rpcDao.saveAndFlush(rpc.getTenantId(), rpc); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, 0L)).isEqualTo(0); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, Long.MAX_VALUE)).isEqualTo(2); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(tenantId, System.currentTimeMillis() + 1)).isEqualTo(1); + int batchSize = 10_000; + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(TenantId.SYS_TENANT_ID, 0L, batchSize)).isEqualTo(0); + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(TenantId.SYS_TENANT_ID, Long.MAX_VALUE, batchSize)).isEqualTo(2); + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(tenantId, System.currentTimeMillis() + 1, batchSize)).isEqualTo(1); } } diff --git a/docker/monitoring/grafana/provisioning/dashboards/transport_connections.json b/docker/monitoring/grafana/provisioning/dashboards/transport_connections.json new file mode 100644 index 0000000000..f092315e94 --- /dev/null +++ b/docker/monitoring/grafana/provisioning/dashboards/transport_connections.json @@ -0,0 +1,458 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-orange", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Total connections" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 19, + "x": 0, + "y": 0 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (protocol) (thingsboard_transport_connections_active{})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{protocol}}", + "refId": "A" + } + ], + "title": "Active Connections by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(209, 138, 78, 1)", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 5, + "x": 19, + "y": 0 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum (thingsboard_transport_connections_active)", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Total Connections", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 19, + "x": 0, + "y": 15 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "thingsboard_transport_connections_active", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{protocol}} {{pod_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Active Connections by Pod", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(209, 138, 78, 1)", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Connections" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "gauge" + } + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 23, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (pod_name) (thingsboard_transport_connections_active)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "__auto", + "queryType": "randomWalk", + "refId": "A" + } + ], + "title": "Connection Count by Pod", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "app": true, + "controller_revision_hash": true, + "instance": true, + "job": true, + "kubernetes_name": true, + "kubernetes_namespace": true, + "node_name": true, + "pod_name": false, + "statefulset_kubernetes_io_pod_name": true + }, + "indexByName": {}, + "renameByName": { + "Time": "", + "Value": "Connections", + "pod_name": "Pod" + } + } + }, + { + "id": "sortBy", + "options": { + "fields": {}, + "sort": [ + { + "field": "Pod" + } + ] + } + } + ], + "type": "table" + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "thingsboard" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Transport Connections", + "uid": "lewbrlwjerwkj3", + "version": 1, + "weekStart": "" +} diff --git a/edqs/src/main/resources/edqs.yml b/edqs/src/main/resources/edqs.yml index 010994dc88..93914fffbe 100644 --- a/edqs/src/main/resources/edqs.yml +++ b/edqs/src/main/resources/edqs.yml @@ -25,19 +25,19 @@ app: # Application version version: "@project.version@" -# Zookeeper connection parameters +# ZooKeeper connection parameters zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:true}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index be49ece5a6..3a08495069 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -14,46 +14,56 @@ # limitations under the License. # -queue_type: "kafka" -request_topic: "js_eval.requests" -queue_prefix: "" -http_port: "8888" # /livenessProbe +# Queue configuration +# Defines the messaging queue backend used to dispatch JS evaluation requests to the executor and receive results. +queue_type: "kafka" # Queue type used for JS evaluation requests: kafka or in-memory +request_topic: "js_eval.requests" # Kafka topic name for incoming JS evaluation requests +queue_prefix: "" # Optional prefix applied to all queue/topic names; useful for environment isolation +http_port: "8888" # HTTP port exposing the /livenessProbe health check endpoint +# JavaScript execution configuration +# Controls response polling behavior and output size limits for individual JS script executions. js: - response_poll_interval: "25" - max_result_size: "300000" + response_poll_interval: "25" # Interval in milliseconds to poll for JS execution responses + max_result_size: "300000" # Maximum allowed size in bytes of a JS execution result +# Kafka configuration +# Connection, producer, and topic settings for the Kafka message broker used as the JS evaluation queue. kafka: bootstrap: # Kafka Bootstrap Servers servers: "localhost:9092" - replication_factor: "1" + replication_factor: "1" # Number of replicas for each Kafka topic partition acks: "1" # -1 = all; 0 = no acknowledgments; 1 = only waits for the leader to acknowledge - batch_size: "128" # for producer - linger_ms: "5" # for producer + batch_size: "128" # Maximum number of messages to accumulate before triggering a batch send + linger_ms: "5" # Time in milliseconds the producer waits before sending a batch, allowing more messages to accumulate partitions_consumed_concurrently: "1" # (EXPERIMENTAL) increase this value if you are planning to handle more than one partition (scale up, scale down) - this will decrease the latency - requestTimeout: "30000" # The default value in kafkajs is: 30000 - connectionTimeout: "1000" # The default value in kafkajs is: 1000 - compression: "none" # gzip or uncompressed - topic_properties: "retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1" - use_confluent_cloud: false - client_id: "kafkajs" #inject pod name to easy identify the client using /opt/kafka/bin/kafka-consumer-groups.sh + requestTimeout: "30000" # Time in milliseconds to wait for a response to a Kafka request before timing out + connectionTimeout: "1000" # Time in milliseconds to wait when establishing a connection to a Kafka broker + compression: "none" # Message compression codec for the producer: none, gzip + topic_properties: "retention.ms:604800000;segment.bytes:52428800;retention.bytes:104857600;partitions:100;min.insync.replicas:1" # Semicolon-separated Kafka topic configuration properties applied on topic creation + use_confluent_cloud: false # Set to true to enable Confluent Cloud-specific configuration (SSL + SASL) + client_id: "kafkajs" # Kafka client identifier; inject the pod name to easily identify the client via kafka-consumer-groups.sh ssl: - enabled: false + enabled: false # Enable SSL/TLS for Kafka broker connections confluent: sasl: - mechanism: "PLAIN" + mechanism: "PLAIN" # SASL mechanism for Confluent Cloud authentication: PLAIN, SCRAM-SHA-256, or SCRAM-SHA-512 +# Logging configuration +# Controls log verbosity, output directory, and log file naming pattern for the JS executor service. logger: - level: "info" - path: "logs" - filename: "tb-js-executor-%DATE%.log" + level: "info" # Log level: trace, debug, info, warn, or error + path: "logs" # Directory path where log files are written + filename: "tb-js-executor-%DATE%.log" # Log file name pattern; %DATE% is replaced with the current date +# JavaScript execution and monitoring configuration +# Sandboxing, script caching, and observability settings for the JS script runtime. script: - use_sandbox: "true" - memory_usage_trace_frequency: "1000" - script_body_trace_frequency: "10000" - stat_print_frequency: "10000" - max_active_scripts: "1000" - slow_query_log_ms: "5.000000" #millis - slow_query_log_body: "false" + use_sandbox: "true" # Run scripts inside an isolated sandbox to prevent access to Node.js internals + memory_usage_trace_frequency: "1000" # Log memory usage every N script executions + script_body_trace_frequency: "10000" # Log the script body every N script executions + stat_print_frequency: "10000" # Print execution statistics every N script executions + max_active_scripts: "1000" # Maximum number of compiled scripts kept in the cache + slow_query_log_ms: "5.000000" # Execution time threshold in milliseconds above which a script is considered slow and logged + slow_query_log_body: "false" # Log the script body when execution time exceeds slow_query_log_ms threshold diff --git a/msa/vc-executor/src/main/resources/tb-vc-executor.yml b/msa/vc-executor/src/main/resources/tb-vc-executor.yml index 56e15d4b4d..188583dc7e 100644 --- a/msa/vc-executor/src/main/resources/tb-vc-executor.yml +++ b/msa/vc-executor/src/main/resources/tb-vc-executor.yml @@ -15,30 +15,33 @@ # # Spring common parameters +# Controls Spring Boot startup mode, web environment type, and bean wiring behaviour. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the embedded HTTP server bind address and port used for actuator and metrics endpoints. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). port: "${HTTP_BIND_PORT:8086}" -# Zookeeper connection parameters. Used for service discovery. +# ZooKeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:true}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Queue configuration parameters +# Defines the message queue type (Kafka or in-memory) and all related producer, consumer, and topic settings. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -196,6 +200,7 @@ queue: msg-chunk-size: "${TB_QUEUE_VC_MSG_CHUNK_SIZE:250000}" # Version control parameters +# Configures the Git-based version control executor, including thread pools and local repository storage. vc: # Pool size for handling export tasks thread_pool_size: "${TB_VC_POOL_SIZE:6}" @@ -206,6 +211,7 @@ vc: repositories-folder: "${TB_VC_GIT_REPOSITORIES_FOLDER:${java.io.tmpdir}/repositories}" # Usage statistics parameters +# Controls collection and reporting of API usage statistics at system, tenant, and customer levels. usage: stats: report: @@ -221,6 +227,7 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables actuator-based metrics collection and configures histogram/timer percentile exports. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" @@ -229,6 +236,7 @@ metrics: percentiles: "${METRICS_TIMER_PERCENTILES:0.5}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, including the metrics and info endpoints. management: endpoints: web: @@ -237,12 +245,14 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Service common properties +# Identifies this microservice instance by type and optional unique ID for cluster coordination. service: type: "${TB_SERVICE_TYPE:tb-vc-executor}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Notification system parameters +# Defines deduplication rules for notification triggers to prevent repeated alerts within a time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index 6856da9233..c23aaabf80 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -125,7 +125,6 @@ org.apache.maven.plugins maven-jar-plugin - 3.1.1 diff --git a/pom.xml b/pom.xml index 45997145c0..0021b3b1d0 100755 --- a/pom.xml +++ b/pom.xml @@ -72,7 +72,7 @@ 1.2.5 1.7.1 3.5.4 - 3.4.0 + 3.5.0 2.8.8TB 2.2.30 0.8 @@ -636,7 +636,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.1.1 + ${jar-plugin.version} org.apache.maven.plugins @@ -656,7 +656,7 @@ com.github.eirslett frontend-maven-plugin - 1.12.0 + 2.0.0 org.apache.maven.plugins diff --git a/transport/coap/src/main/resources/tb-coap-transport.yml b/transport/coap/src/main/resources/tb-coap-transport.yml index 376075a0a4..c618a921aa 100644 --- a/transport/coap/src/main/resources/tb-coap-transport.yml +++ b/transport/coap/src/main/resources/tb-coap-transport.yml @@ -15,30 +15,33 @@ # # Spring common parameters +# Controls core Spring Boot application settings such as web environment and application type. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the HTTP server bind address and port for the actuator/metrics web endpoint. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). port: "${HTTP_BIND_PORT:8083}" -# Zookeeper connection parameters. Used for service discovery. +# ZooKeeper connection parameters. Used for service discovery. +# Defines how this service registers itself and discovers other nodes via Apache ZooKeeper. zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Configures the caching backend and per-entity cache settings used by the transport service. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +59,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Defines connection mode, authentication, SSL, and connection pool settings for Redis or Valkey. redis: connection: # standalone or cluster or sentinel @@ -90,9 +95,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -172,6 +177,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # CoAP server parameters +# Configures the CoAP and CoAP/DTLS server endpoints, including bind addresses, timeouts, and TLS credentials. coap: # CoAP bind-address bind_address: "${COAP_BIND_ADDRESS:0.0.0.0}" @@ -253,6 +259,7 @@ coap: dtls_session_report_timeout: "${TB_COAP_X509_DTLS_SESSION_REPORT_TIMEOUT:1800000}" # Queue configuration parameters +# Defines the message queue backend (Kafka) and all topic, partition, and consumer settings for inter-service communication. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -406,12 +413,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common properties +# Identifies this service instance within the ThingsBoard cluster. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage metrics at system, tenant, and customer levels. usage: stats: report: @@ -427,11 +436,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables Micrometer/Actuator metrics collection and configures the exposed management endpoints. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure for health checks and metrics scraping. management: endpoints: web: @@ -440,6 +451,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures deduplication rules for platform notifications to prevent repeated alerts within a time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/http/src/main/resources/tb-http-transport.yml b/transport/http/src/main/resources/tb-http-transport.yml index 1f6a251324..5173f5c26d 100644 --- a/transport/http/src/main/resources/tb-http-transport.yml +++ b/transport/http/src/main/resources/tb-http-transport.yml @@ -15,6 +15,7 @@ # # Server common properties +# Configures the HTTP server bind address, port, SSL/TLS settings, and HTTP/2 support. server: # Server bind address address: "${HTTP_BIND_ADDRESS:0.0.0.0}" @@ -54,23 +55,25 @@ server: enabled: "${HTTP2_ENABLED:true}" # Spring common parameters +# Controls Spring Boot framework-level settings such as circular references and multipart upload limits. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. spring.servlet.multipart.max-file-size: "${SPRING_SERVLET_MULTIPART_MAX_FILE_SIZE:50MB}" # Total file size cannot exceed 50MB when configuring file uploads spring.servlet.multipart.max-request-size: "${SPRING_SERVLET_MULTIPART_MAX_REQUEST_SIZE:50MB}" # Total request size for a multipart/form-data cannot exceed 50MB -# Zookeeper connection parameters. Used for service discovery. +# ZooKeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. @@ -78,6 +81,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Defines the cache backend type and per-entity cache configuration such as TTL and maximum size. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -87,6 +91,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Configures connection mode (standalone, cluster, sentinel), credentials, SSL, and connection pool settings. redis: # standalone or cluster or sentinel connection: @@ -123,9 +128,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -166,6 +171,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # HTTP server parameters +# Configures HTTP transport request timeouts, payload limits, session management, JSON processing, logging, and statistics. transport: http: # HTTP request processing timeout in milliseconds @@ -203,6 +209,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # Queue configuration parameters +# Configures the messaging queue backend (Kafka), topic definitions, partitioning, and per-service consumer settings. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka) . @@ -355,12 +362,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # General service parameters +# Defines the service type and unique identifier used for service discovery and cluster coordination. service: type: "${TB_SERVICE_TYPE:tb-transport}" # type of service # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage statistics at the system, tenant, and optional customer level. usage: stats: report: @@ -376,11 +385,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables actuator metrics collection and exposure for monitoring integrations such as Prometheus. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator management endpoints, including which endpoints are exposed over HTTP. management: endpoints: web: @@ -389,6 +400,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures notification rules, including deduplication windows to prevent repeated alerts for the same trigger. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml index b4d8d6c7eb..2711eae3ec 100644 --- a/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml +++ b/transport/lwm2m/src/main/resources/tb-lwm2m-transport.yml @@ -15,30 +15,33 @@ # # Spring common parameters +# Controls Spring Boot web environment and application type settings used for optional metrics exposure. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# HTTP server bind address and port, effective only when the web environment is enabled. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). port: "${HTTP_BIND_PORT:8083}" -# Zookeeper connection parameters. Used for service discovery. +# ZooKeeper connection parameters +# Controls ZooKeeper-based service discovery and cluster coordination for microservice deployments. zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Configures the cache backend type and per-entity cache specifications such as TTL and maximum size. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +59,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Connection, authentication, SSL, and connection pool settings for the Redis or Valkey cache backend. redis: connection: # standalone or cluster or sentinel @@ -90,9 +95,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -133,6 +138,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # LWM2M server parameters +# Session, JSON processing, RPC, logging, and LwM2M-specific transport settings including DTLS, bootstrap, and OTA. transport: sessions: # Session inactivity timeout is a global configuration parameter that defines how long the device transport session will be opened after the last message arrives from the device. @@ -303,6 +309,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # Queue configuration properties +# Defines the message queue backend (Kafka) and per-topic settings for transport, core, rule-engine, and JS evaluation. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -456,12 +463,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common parameters +# Identifies the service type and its unique instance ID within a ThingsBoard cluster. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage statistics at system, tenant, and optionally customer level. usage: stats: report: @@ -477,11 +486,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables Actuator-based metrics collection (e.g., Prometheus endpoint). metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, e.g. for Prometheus metrics scraping. management: endpoints: web: @@ -490,6 +501,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures deduplication rules for notification triggers to prevent repeated alerts within a time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml index 0b96c57c13..02c451fd69 100644 --- a/transport/mqtt/src/main/resources/tb-mqtt-transport.yml +++ b/transport/mqtt/src/main/resources/tb-mqtt-transport.yml @@ -15,30 +15,33 @@ # # Spring common parameters +# Controls Spring Boot application type, web environment, and bean dependency settings. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the embedded HTTP server bind address and port for actuator/metrics endpoints. server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). port: "${HTTP_BIND_PORT:8083}" -# Zookeeper connection parameters. Used for service discovery. +# ZooKeeper connection parameters. Used for service discovery. +# Defines connection, session, retry settings and the node directory path for ZooKeeper-based cluster coordination. zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Configures the cache backend type (Caffeine or Redis) and per-cache TTL and size limits. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +59,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Covers connection mode (standalone, cluster, sentinel), SSL, credentials, and connection pool tuning. redis: # standalone or cluster or sentinel connection: @@ -91,9 +96,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -134,6 +139,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # MQTT server parameters +# Configures the MQTT transport layer including bind address/port, SSL, Netty tuning, session management, rate limits, and message processing options. transport: mqtt: # MQTT bind-address @@ -236,6 +242,7 @@ transport: ip_block_timeout: "${TB_TRANSPORT_IP_BLOCK_TIMEOUT:60000}" # Queue configuration parameters +# Defines the messaging queue backend (Kafka), topic names, partitioning, and poll/processing settings for all platform microservices. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -389,12 +396,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common properties +# Identifies this service instance within the cluster by type and optional unique ID. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls reporting of API usage statistics at system, tenant, and customer levels with configurable intervals. usage: stats: report: @@ -410,11 +419,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables Micrometer actuator metrics collection for monitoring this service. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, including which endpoints are accessible over HTTP. management: endpoints: web: @@ -423,6 +434,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Configures deduplication rules for notification triggers to prevent repeated alerts within a defined time window. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/transport/snmp/src/main/resources/tb-snmp-transport.yml b/transport/snmp/src/main/resources/tb-snmp-transport.yml index 71524ba47e..028e9e2714 100644 --- a/transport/snmp/src/main/resources/tb-snmp-transport.yml +++ b/transport/snmp/src/main/resources/tb-snmp-transport.yml @@ -15,30 +15,33 @@ # # Spring common parameters +# Controls Spring Boot application type and web environment settings for the SNMP transport service. spring.main.web-environment: "${WEB_APPLICATION_ENABLE:false}" # If you enabled process metrics you should also enable 'web-environment'. spring.main.web-application-type: "${WEB_APPLICATION_TYPE:none}" # If you enabled process metrics you should set 'web-application-type' to 'servlet' value. spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. # Server common parameters +# Configures the embedded HTTP server bind address and port (used when web environment is enabled). server: # Server bind address (has no effect if web-environment is disabled). address: "${HTTP_BIND_ADDRESS:0.0.0.0}" # Server bind port (has no effect if web-environment is disabled). port: "${HTTP_BIND_PORT:8083}" -# Zookeeper connection parameters. Used for service discovery. +# ZooKeeper connection parameters. Used for service discovery. +# Defines ZooKeeper connectivity, timeouts, and cluster node registration settings for service coordination. zk: - # Enable/disable zookeeper discovery service. + # Enable/disable ZooKeeper discovery service. enabled: "${ZOOKEEPER_ENABLED:false}" - # Zookeeper connect string + # ZooKeeper connect string url: "${ZOOKEEPER_URL:localhost:2181}" - # Zookeeper retry interval in milliseconds + # ZooKeeper retry interval in milliseconds retry_interval_ms: "${ZOOKEEPER_RETRY_INTERVAL_MS:3000}" - # Zookeeper connection timeout in milliseconds + # ZooKeeper connection timeout in milliseconds connection_timeout_ms: "${ZOOKEEPER_CONNECTION_TIMEOUT_MS:3000}" - # Zookeeper session timeout in milliseconds + # ZooKeeper session timeout in milliseconds session_timeout_ms: "${ZOOKEEPER_SESSION_TIMEOUT_MS:3000}" - # Name of the directory in zookeeper 'filesystem' + # Name of the directory in ZooKeeper 'filesystem' zk_dir: "${ZOOKEEPER_NODES_DIR:/thingsboard}" # The recalculate_delay property is recommended in a microservices architecture setup for rule-engine services. # This property provides a pause to ensure that when a rule-engine service is restarted, other nodes don't immediately attempt to recalculate their partitions. @@ -46,6 +49,7 @@ zk: recalculate_delay: "${ZOOKEEPER_RECALCULATE_DELAY_MS:0}" # Cache parameters +# Specifies the caching backend (Caffeine or Redis) and per-entity cache TTL and size limits. cache: # caffeine or redis type: "${CACHE_TYPE:redis}" @@ -55,6 +59,7 @@ cache: maxSize: "${CACHE_SPECS_ENTITY_LIMITS_MAX_SIZE:100000}" # 0 means the cache is disabled # Redis/Valkey configuration parameters +# Covers standalone, cluster, and sentinel Redis connection modes, SSL, authentication, and connection pool tuning. redis: connection: # standalone or cluster or sentinel @@ -90,9 +95,9 @@ redis: password: "${REDIS_SENTINEL_PASSWORD:}" # if set false will be used pool config build from values of the pool config section useDefaultPoolConfig: "${REDIS_USE_DEFAULT_POOL_CONFIG:true}" - # db index + # Redis logical database index to select after connecting. db: "${REDIS_DB:0}" - # db password + # Password for Redis authentication (leave empty if not required). password: "${REDIS_PASSWORD:}" # Redis username for ACL authentication (Redis 6.0+). Leave empty for legacy password-only auth username: "${REDIS_USERNAME:}" @@ -133,6 +138,7 @@ redis: blockWhenExhausted: "${REDIS_POOL_CONFIG_BLOCK_WHEN_EXHAUSTED:true}" # Snmp server parameters +# Configures the SNMP transport protocol, bind port, PDU limits, session management, JSON handling, logging, and statistics. transport: snmp: # Enable/disable SNMP transport protocol @@ -184,6 +190,7 @@ transport: print-interval-ms: "${TB_TRANSPORT_STATS_PRINT_INTERVAL_MS:60000}" # Queue configuration parameters +# Defines the message queue backend (Kafka) and topic/consumer settings for transport, core, rule-engine, and JS evaluation. queue: type: "${TB_QUEUE_TYPE:kafka}" # kafka (Apache Kafka) prefix: "${TB_QUEUE_PREFIX:}" # Global queue prefix. If specified, prefix is added before default topic name: 'prefix.default_topic_name'. Prefix is applied to all topics (and consumer groups for kafka). @@ -344,12 +351,14 @@ queue: poll_interval: "${TB_QUEUE_TRANSPORT_NOTIFICATIONS_POLL_INTERVAL_MS:25}" # Service common parameters +# Identifies this service instance by type and unique ID within a ThingsBoard cluster. service: type: "${TB_SERVICE_TYPE:tb-transport}" # service type # Unique id for this service (autogenerated if empty) id: "${TB_SERVICE_ID:}" # Usage statistics parameters +# Controls collection and reporting intervals for API usage statistics at system, tenant, and customer levels. usage: stats: report: @@ -365,11 +374,13 @@ usage: pack_size: "${USAGE_STATS_REPORT_PACK_SIZE:1024}" # Metrics parameters +# Enables or disables actuator-based metrics collection (e.g., Prometheus) for the SNMP transport service. metrics: # Enable/disable actuator metrics. enabled: "${METRICS_ENABLED:false}" # General management parameters +# Configures Spring Boot Actuator endpoint exposure, including which management endpoints are accessible over HTTP. management: endpoints: web: @@ -378,6 +389,7 @@ management: include: "${METRICS_ENDPOINTS_EXPOSE:info}" # Notification system parameters +# Defines deduplication rules for notification triggers to prevent duplicate alerts within configured time windows. notification_system: rules: # Semicolon-separated deduplication durations (in millis) for trigger types. Format: 'NotificationRuleTriggerType1:123;NotificationRuleTriggerType2:456' diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html index c9247f5549..553349df8e 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html @@ -93,7 +93,8 @@ (click)="deleteTelemetry($event)"> delete -