diff --git a/.gitignore b/.gitignore index 777a52efc7..ee8c78dad3 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ rebuild-docker.sh */.run/** .run/** .run -.claude/ +.claude +.planning diff --git a/TEST_FAST.md b/TEST_FAST.md index 4d1e9dc9a7..eb2013c601 100644 --- a/TEST_FAST.md +++ b/TEST_FAST.md @@ -6,7 +6,9 @@ export MAVEN_OPTS="-Xmx1024m" export NODE_OPTIONS="--max_old_space_size=4096" export SUREFIRE_JAVA_OPTS="-Xmx1200m -Xss256k -XX:+ExitOnOutOfMemoryError" -mvn clean install -T6 -DskipTests +# Compile and install all modules, skip packaging artifacts not needed for tests +mvn clean install -T6 -DskipTests -Dpkg.skip=true + mvn test -pl='!application,!dao,!ui-ngx,!msa/js-executor,!msa/web-ui' -T4 mvn test -pl dao -Dparallel=packages -DforkCount=4 @@ -31,6 +33,18 @@ mvn test -pl application -Dtest=' ' -DforkCount=6 -Dparallel=packages -Dsurefire.rerunFailingTestsCount=2 -Dsurefire.failOnFlakeCount=5 ``` +## pkg.skip.* flags reference + +Use `-Dpkg.skip=true` to skip all packaging at once (equivalent to all four flags below). + +| Flag | Skips | Safe to skip for tests? | +|----------------------------|-------------------------------------------|--------------------------------------------------------------| +| `-Dpkg.skip=true` | All of the below (bootjar + deb + rpm + zip) | Yes | +| `-Dpkg.skip.bootjar=true` | `spring-boot:repackage` (`*-boot.jar`) | Yes — tests use the regular `.jar`, not the fat boot jar | +| `-Dpkg.skip.deb=true` | Gradle `buildDeb` + Maven `attach-artifact` | Yes — MSA docker modules copy the DEB from `target/` directly | +| `-Dpkg.skip.rpm=true` | Gradle `buildRpm` | Yes — no test depends on the RPM | +| `-Dpkg.skip.zip=true` | `maven-assembly-plugin` Windows ZIP | Yes — no test depends on the ZIP | + ## Testcontainers compatibility with the Docker API workaround In case your tests failed to run testcontainers due to unsupported Docker API version diff --git a/application/pom.xml b/application/pom.xml index cbbf8bedb3..af5d26070d 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard application @@ -35,7 +35,6 @@ UTF-8 ${basedir}/.. java - false process-resources package thingsboard @@ -493,10 +492,6 @@ org.apache.maven.plugins maven-assembly-plugin - - org.apache.maven.plugins - maven-install-plugin - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java index 892eb2beda..a9692f4b9d 100644 --- a/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java +++ b/application/src/main/java/org/thingsboard/server/actors/ActorSystemContext.java @@ -615,11 +615,21 @@ public class ActorSystemContext { @Value("${actors.rule.external.ssrf_additional_blocked_hosts:}") private List ssrfAdditionalBlockedHosts; + @Value("${actors.rule.external.ssrf_allowed_hosts:}") + private List ssrfAllowedHosts; + @PostConstruct public void init() { this.localCacheType = "caffeine".equals(cacheType); SsrfProtectionValidator.setEnabled(ssrfProtectionEnabled); SsrfProtectionValidator.setAdditionalBlockedHosts(ssrfAdditionalBlockedHosts); + SsrfProtectionValidator.setAllowedHosts(ssrfAllowedHosts); + if (!ssrfProtectionEnabled) { + log.warn("SSRF protection for external rule nodes is DISABLED. This allows rule chains to make HTTP requests to " + + "internal/private network addresses including cloud metadata endpoints. It is strongly recommended to " + + "enable SSRF protection by setting SSRF_PROTECTION_ENABLED=true. If your rule chains need to access " + + "devices on local networks, use SSRF_ALLOWED_HOSTS to whitelist specific addresses or ranges."); + } } @Value("${actors.tenant.create_components_on_init:true}") diff --git a/application/src/main/java/org/thingsboard/server/config/HttpSecurityHeadersCustomizer.java b/application/src/main/java/org/thingsboard/server/config/HttpSecurityHeadersCustomizer.java new file mode 100644 index 0000000000..318c435353 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/HttpSecurityHeadersCustomizer.java @@ -0,0 +1,64 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.security.config.annotation.web.configurers.HeadersConfigurer; +import org.springframework.security.web.header.writers.StaticHeadersWriter; +import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; + +@Slf4j +@Component +@RequiredArgsConstructor +public class HttpSecurityHeadersCustomizer { + + private final HttpSecurityHeadersProperties properties; + + public void customize(HeadersConfigurer headers) { + if (properties.getXContentTypeOptions().isEnabled()) { + headers.contentTypeOptions(config -> {}); + } + + if (properties.getReferrerPolicy().isEnabled()) { + headers.addHeaderWriter(new StaticHeadersWriter("Referrer-Policy", properties.getReferrerPolicy().getValue())); + } + + if (properties.getXFrameOptions().isEnabled()) { + String value = properties.getXFrameOptions().getValue(); + if ("DENY".equalsIgnoreCase(value)) { + headers.frameOptions(HeadersConfigurer.FrameOptionsConfig::deny); + } else { + if (!"SAMEORIGIN".equalsIgnoreCase(value)) { + log.warn("Unrecognized X-Frame-Options value '{}', falling back to SAMEORIGIN. Valid values: DENY, SAMEORIGIN", value); + } + headers.frameOptions(HeadersConfigurer.FrameOptionsConfig::sameOrigin); + } + } + + if (properties.getContentSecurityPolicy().isEnabled() && StringUtils.hasText(properties.getContentSecurityPolicy().getValue())) { + headers.contentSecurityPolicy(csp -> { + csp.policyDirectives(properties.getContentSecurityPolicy().getValue()); + if (properties.getContentSecurityPolicy().isReportOnly()) { + csp.reportOnly(); + } + }); + } + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/config/HttpSecurityHeadersProperties.java b/application/src/main/java/org/thingsboard/server/config/HttpSecurityHeadersProperties.java new file mode 100644 index 0000000000..224e2aeea0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/HttpSecurityHeadersProperties.java @@ -0,0 +1,56 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationProperties(prefix = "security.headers") +@Data +public class HttpSecurityHeadersProperties { + + private XContentTypeOptions xContentTypeOptions = new XContentTypeOptions(); + private ReferrerPolicy referrerPolicy = new ReferrerPolicy(); + private XFrameOptions xFrameOptions = new XFrameOptions(); + private ContentSecurityPolicy contentSecurityPolicy = new ContentSecurityPolicy(); + + @Data + public static class XContentTypeOptions { + private boolean enabled = true; + } + + @Data + public static class ReferrerPolicy { + private boolean enabled = true; + private String value = "strict-origin-when-cross-origin"; + } + + @Data + public static class XFrameOptions { + private boolean enabled = false; + private String value = "SAMEORIGIN"; + } + + @Data + public static class ContentSecurityPolicy { + private boolean enabled = false; + private String value = ""; + private boolean reportOnly = false; + } + +} diff --git a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java index 1f94afb398..f3efaf8d4e 100644 --- a/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/TbRuleEngineSecurityConfiguration.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.config; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.boot.autoconfigure.security.SecurityProperties; import org.springframework.context.annotation.Bean; @@ -33,11 +34,16 @@ import org.springframework.security.web.SecurityFilterChain; @ConditionalOnExpression("'${service.type:null}'=='tb-rule-engine'") public class TbRuleEngineSecurityConfiguration { + @Autowired + private HttpSecurityHeadersCustomizer httpSecurityHeadersCustomizer; + @Bean SecurityFilterChain filterChain(HttpSecurity http) throws Exception { - http.headers(headers -> headers - .cacheControl(config -> {}) - .frameOptions(config -> {}).disable()) + http.headers(headers -> { + headers.defaultsDisabled(); + headers.cacheControl(config -> {}); + httpSecurityHeadersCustomizer.customize(headers); + }) .cors(cors -> {}) .csrf(AbstractHttpConfigurer::disable) .authorizeHttpRequests(config -> config diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index cac4f4165e..712224b5d2 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -131,6 +131,9 @@ public class ThingsboardSecurityConfiguration { @Autowired private AuthExceptionHandler authExceptionHandler; + @Autowired + private HttpSecurityHeadersCustomizer httpSecurityHeadersCustomizer; + @Bean protected PayloadSizeFilter payloadSizeFilter() { return new PayloadSizeFilter(maxPayloadSizeConfig); @@ -198,9 +201,11 @@ public class ThingsboardSecurityConfiguration { http .securityMatchers(matchers -> matchers .requestMatchers("/*.js", "/*.css", "/*.ico", "/assets/**", "/static/**")) - .headers(header -> header - .defaultsDisabled() - .addHeaderWriter(new StaticHeadersWriter(HttpHeaders.CACHE_CONTROL, "max-age=0, public"))) + .headers(headers -> { + headers.defaultsDisabled(); + headers.addHeaderWriter(new StaticHeadersWriter(HttpHeaders.CACHE_CONTROL, "max-age=0, public")); + httpSecurityHeadersCustomizer.customize(headers); + }) .authorizeHttpRequests((authorize) -> authorize.anyRequest().permitAll()) .requestCache(RequestCacheConfigurer::disable) .securityContext(AbstractHttpConfigurer::disable) @@ -210,9 +215,11 @@ public class ThingsboardSecurityConfiguration { @Bean SecurityFilterChain filterChain(HttpSecurity http) throws Exception { - http.headers(headers -> headers - .cacheControl(config -> {}) - .frameOptions(config -> {}).disable()) + http.headers(headers -> { + headers.defaultsDisabled(); + headers.cacheControl(config -> {}); + httpSecurityHeadersCustomizer.customize(headers); + }) .cors(cors -> {}) .csrf(AbstractHttpConfigurer::disable) .exceptionHandling(config -> {}) diff --git a/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java b/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java index 3dd15b7255..bcf897f8fb 100644 --- a/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java +++ b/application/src/main/java/org/thingsboard/server/exception/ThingsboardErrorResponseHandler.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.exception; +import jakarta.persistence.PersistenceException; import jakarta.servlet.RequestDispatcher; import jakarta.servlet.ServletException; import jakarta.servlet.http.HttpServletRequest; @@ -50,6 +51,7 @@ import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.msg.tools.MaxPayloadSizeExceededException; import org.thingsboard.server.common.msg.tools.TbRateLimitsException; +import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.service.security.exception.AuthMethodNotSupportedException; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; import org.thingsboard.server.service.security.exception.UserPasswordExpiredException; @@ -154,8 +156,8 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand handleAuthenticationException((AuthenticationException) exception, response); } else if (exception instanceof MaxPayloadSizeExceededException) { handleMaxPayloadSizeExceededException(response, (MaxPayloadSizeExceededException) exception); - } else if (exception instanceof DataAccessException e) { - handleDatabaseException(e, response); + } else if (exception instanceof DataAccessException || exception instanceof PersistenceException) { + handleDatabaseException(exception, response); } else { response.setStatus(HttpStatus.INTERNAL_SERVER_ERROR.value()); JacksonUtil.writeValue(response.getWriter(), ThingsboardErrorResponse.of(exception.getMessage(), @@ -209,8 +211,17 @@ public class ThingsboardErrorResponseHandler extends ResponseEntityExceptionHand private void handleDatabaseException(Throwable databaseException, HttpServletResponse response) throws IOException { ThingsboardErrorResponse errorResponse; - if (databaseException instanceof ConstraintViolationException) { - errorResponse = ThingsboardErrorResponse.of(ExceptionUtils.getRootCause(databaseException).getMessage(), ThingsboardErrorCode.BAD_REQUEST_PARAMS, HttpStatus.BAD_REQUEST); + ConstraintViolationException constraintViolationException = DaoUtil.extractConstraintViolation(databaseException); + if (constraintViolationException != null) { + log.debug("Constraint violation: {}", ExceptionUtils.getRootCauseMessage(databaseException)); + String constraintName = constraintViolationException.getConstraintName(); + String userMessage; + if (constraintName != null && !constraintName.isEmpty()) { + userMessage = "Constraint violation: " + constraintName; + } else { + userMessage = "Constraint violation"; + } + errorResponse = ThingsboardErrorResponse.of(userMessage, ThingsboardErrorCode.BAD_REQUEST_PARAMS, HttpStatus.BAD_REQUEST); } else { log.warn("Database error: {} - {}", databaseException.getClass().getSimpleName(), ExceptionUtils.getRootCauseMessage(databaseException)); errorResponse = ThingsboardErrorResponse.of("Database error", ThingsboardErrorCode.DATABASE, HttpStatus.INTERNAL_SERVER_ERROR); diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java index 03228c52d4..e1211c53a7 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcService.java @@ -19,7 +19,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContext; +import io.grpc.netty.shaded.io.netty.handler.ssl.SslContextBuilder; import io.grpc.stub.StreamObserver; import jakarta.annotation.Nullable; import jakarta.annotation.PreDestroy; @@ -37,7 +40,8 @@ import org.thingsboard.server.cache.TbTransactionalCache; import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AttributeScope; import org.thingsboard.server.common.data.DataConstants; -import org.thingsboard.server.common.data.ResourceUtils; +import org.thingsboard.server.common.data.StringUtils; +import org.thingsboard.server.common.transport.config.ssl.PemSslCredentials; import org.thingsboard.server.common.data.edge.Edge; import org.thingsboard.server.common.data.edge.EdgeEvent; import org.thingsboard.server.common.data.id.EdgeId; @@ -67,7 +71,6 @@ import org.thingsboard.server.service.edge.EdgeContextComponent; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -112,6 +115,8 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i private String certFileResource; @Value("${edges.rpc.ssl.private_key}") private String privateKeyResource; + @Value("${edges.rpc.ssl.key_password:}") + private String keyPassword; @Value("${edges.state.persistToTelemetry:false}") private boolean persistToTelemetry; @Value("${edges.rpc.client_max_keep_alive_time_sec:1}") @@ -176,9 +181,7 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i .addService(this); if (sslEnabled) { try { - InputStream certFileIs = ResourceUtils.getInputStream(this, certFileResource); - InputStream privateKeyFileIs = ResourceUtils.getInputStream(this, privateKeyResource); - builder.useTransportSecurity(certFileIs, privateKeyFileIs); + setupSsl(builder); } catch (Exception e) { log.error("Unable to set up SSL context. Reason: " + e.getMessage(), e); throw new RuntimeException("Unable to set up SSL context!", e); @@ -199,6 +202,33 @@ public class EdgeGrpcService extends EdgeRpcServiceGrpc.EdgeRpcServiceImplBase i log.info("Edge RPC service initialized!"); } + /** + * Configures TLS for the Edge gRPC server. + *

+ * Delegates PEM parsing and key management to {@link PemSslCredentials} — the same + * class used by MQTT, CoAP, and LwM2M transports — which supports: + *

    + *
  • Separate certificate and private key files (classic two-file setup)
  • + *
  • Combined PEM: certificate chain + private key in a single {@code cert} file + * ({@code private_key} left empty)
  • + *
  • Encrypted private keys (password supplied via {@code key_password})
  • + *
+ * Path resolution (for both {@code cert} and {@code private_key}) is handled by + * {@link org.thingsboard.server.common.data.ResourceUtils#getInputStream ResourceUtils}: + * absolute path → relative / working-dir → classpath → {@code classpath:} prefix. + */ + void setupSsl(NettyServerBuilder builder) throws Exception { + PemSslCredentials credentials = new PemSslCredentials(); + credentials.setCertFile(certFileResource); + credentials.setKeyFile(StringUtils.isEmpty(privateKeyResource) ? null : privateKeyResource); + credentials.setKeyPassword(keyPassword); + credentials.init(false); + + SslContext sslContext = GrpcSslContexts.configure( + SslContextBuilder.forServer(credentials.createKeyManagerFactory())).build(); + builder.sslContext(sslContext); + } + @PreDestroy public void destroy() { if (server != null) { diff --git a/application/src/main/java/org/thingsboard/server/service/notification/channels/MicrosoftTeamsNotificationChannel.java b/application/src/main/java/org/thingsboard/server/service/notification/channels/MicrosoftTeamsNotificationChannel.java index df163283ce..d023ed9b53 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/channels/MicrosoftTeamsNotificationChannel.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/channels/MicrosoftTeamsNotificationChannel.java @@ -29,6 +29,7 @@ import org.springframework.http.MediaType; import org.springframework.stereotype.Component; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.SsrfProtectionValidator; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.NotificationDeliveryMethod; import org.thingsboard.server.common.data.notification.info.NotificationInfo; @@ -109,10 +110,13 @@ public class MicrosoftTeamsNotificationChannel implements NotificationChannel request = new HttpEntity<>(JacksonUtil.toString(teamsAdaptiveCard), headers); - restTemplate.postForEntity(new URI(targetConfig.getWebhookUrl()), request, String.class); + restTemplate.postForEntity(webhookUri, request, String.class); } private void sendTeamsMessageCard(MicrosoftTeamsNotificationTargetConfig targetConfig, MicrosoftTeamsDeliveryMethodNotificationTemplate processedTemplate, NotificationProcessingContext ctx) throws JsonProcessingException, URISyntaxException { @@ -139,10 +143,13 @@ public class MicrosoftTeamsNotificationChannel implements NotificationChannel request = new HttpEntity<>(JacksonUtil.toString(teamsMessageCard), headers); - restTemplate.postForEntity(new URI(targetConfig.getWebhookUrl()), request, String.class); + restTemplate.postForEntity(webhookUri, request, String.class); } private String getButtonUri(MicrosoftTeamsDeliveryMethodNotificationTemplate processedTemplate, NotificationProcessingContext ctx) throws JsonProcessingException { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java index 8477c69a99..97d24b8601 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/CustomOAuth2ClientMapper.java @@ -23,12 +23,15 @@ import org.springframework.security.oauth2.client.authentication.OAuth2Authentic import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.SsrfProtectionValidator; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.oauth2.OAuth2CustomMapperConfig; import org.thingsboard.server.common.data.oauth2.OAuth2MapperConfig; import org.thingsboard.server.common.data.oauth2.OAuth2Client; import org.thingsboard.server.dao.oauth2.OAuth2User; import org.thingsboard.server.queue.util.TbCoreComponent; + +import java.net.URI; import org.thingsboard.server.service.security.model.SecurityUser; @Service(value = "customOAuth2ClientMapper") @@ -64,6 +67,7 @@ public class CustomOAuth2ClientMapper extends AbstractOAuth2ClientMapper impleme throw new RuntimeException("Can't convert principal to JSON string", e); } try { + SsrfProtectionValidator.validateUri(new URI(custom.getUrl())); return restTemplate.postForEntity(custom.getUrl(), request, OAuth2User.class).getBody(); } catch (Exception e) { log.error("There was an error during connection to custom mapper endpoint", e); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java index f0f2e1eaa7..fc8c5c3be2 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbEntityDataSubscriptionService.java @@ -34,6 +34,7 @@ import org.springframework.web.socket.CloseStatus; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.dao.nosql.ResultSetSizeLimitExceededException; import org.thingsboard.server.common.data.kv.BaseReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQuery; import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; @@ -242,7 +243,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc @Override public void onFailure(Throwable t) { - log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId()); + log.warn("[{}][{}] Failed to process command", finalCtx.getSessionId(), finalCtx.getCmdId(), t); + if (t instanceof ResultSetSizeLimitExceededException) { + sendError(finalCtx, t); + } } }, wsCallBackExecutor); } @@ -258,7 +262,18 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc handleLatestCmd(ctx, cmd.getLatestCmd()); } if (cmd.getTsCmd() != null) { - handleTimeSeriesCmd(ctx, cmd.getTsCmd()); + Futures.addCallback(handleTimeSeriesCmd(ctx, cmd.getTsCmd()), new FutureCallback<>() { + @Override + public void onSuccess(TbEntityDataSubCtx result) {} + + @Override + public void onFailure(Throwable t) { + log.warn("[{}][{}] Failed to process timeseries command", ctx.getSessionId(), ctx.getCmdId(), t); + if (t instanceof ResultSetSizeLimitExceededException) { + sendError(ctx, t); + } + } + }, wsCallBackExecutor); } } else { checkAndSendInitialData(ctx); @@ -268,6 +283,10 @@ public class DefaultTbEntityDataSubscriptionService implements TbEntityDataSubsc } } + private void sendError(TbEntityDataSubCtx ctx, Throwable t) { + ctx.sendWsMsg(new EntityDataUpdate(ctx.getCmdId(), SubscriptionErrorCode.INTERNAL_ERROR.getCode(), t.getMessage())); + } + private void checkAndSendInitialData(@Nullable TbEntityDataSubCtx theCtx) { if (!theCtx.isInitialDataSent()) { EntityDataUpdate update = new EntityDataUpdate(theCtx.getCmdId(), theCtx.getData(), null, theCtx.getMaxEntitiesPerDataSubscription()); diff --git a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java index 51663eea9c..2b1a810924 100644 --- a/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java +++ b/application/src/main/java/org/thingsboard/server/service/subscription/DefaultTbLocalSubscriptionService.java @@ -348,7 +348,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer if (sub.isLatestValues()) { for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); - if (stateTs == null || kv.getTs() >= stateTs) { + if (stateTs == null || kv.getTs() >= stateTs || kv.isDeletedEntry()) { if (updateData == null) { updateData = new ArrayList<>(); } @@ -362,7 +362,7 @@ public class DefaultTbLocalSubscriptionService implements TbLocalSubscriptionSer for (TsKvEntry kv : data) { Long stateTs = keyStates.get(kv.getKey()); if (stateTs != null) { - if (!sub.isLatestValues() || kv.getTs() >= stateTs) { + if (!sub.isLatestValues() || kv.getTs() >= stateTs || kv.isDeletedEntry()) { if (updateData == null) { updateData = new ArrayList<>(); } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java index 5865d9e39d..596f5a8754 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/AbstractCleanUpService.java @@ -32,4 +32,8 @@ public abstract class AbstractCleanUpService { return partitionService.resolve(ServiceType.TB_CORE, TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID).isMyPartition(); } + protected boolean isTenantPartitionMine(TenantId tenantId) { + return partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition(); + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java index ddc95b74e9..83855bc8f8 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/NotificationsCleanUpService.java @@ -20,33 +20,41 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.NotificationRequestConfig; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.dao.notification.NotificationRequestDao; import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; +import java.time.Instant; import java.util.concurrent.TimeUnit; import static org.thingsboard.server.dao.model.ModelConstants.NOTIFICATION_TABLE_NAME; +@Slf4j @Service @ConditionalOnExpression("${sql.ttl.notifications.enabled:true} && ${sql.ttl.notifications.ttl:0} > 0") -@Slf4j public class NotificationsCleanUpService extends AbstractCleanUpService { private final SqlPartitioningRepository partitioningRepository; private final NotificationRequestDao notificationRequestDao; + private final TenantService tenantService; @Value("${sql.ttl.notifications.ttl:2592000}") private long ttlInSec; @Value("${sql.notifications.partition_size:168}") private int partitionSizeInHours; + @Value("${sql.ttl.notifications.removal_batch_size:10000}") + private int removalBatchSize; public NotificationsCleanUpService(PartitionService partitionService, SqlPartitioningRepository partitioningRepository, - NotificationRequestDao notificationRequestDao) { + NotificationRequestDao notificationRequestDao, TenantService tenantService) { super(partitionService); this.partitioningRepository = partitioningRepository; this.notificationRequestDao = notificationRequestDao; + this.tenantService = tenantService; } @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.notifications.checking_interval_ms:86400000})}", @@ -54,18 +62,65 @@ public class NotificationsCleanUpService extends AbstractCleanUpService { public void cleanUp() { long expTime = System.currentTimeMillis() - TimeUnit.SECONDS.toMillis(ttlInSec); long partitionDurationMs = TimeUnit.HOURS.toMillis(partitionSizeInHours); - if (!isSystemTenantPartitionMine()) { + if (isSystemTenantPartitionMine()) { + partitioningRepository.dropPartitionsBefore(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); + } else { partitioningRepository.cleanupPartitionsCache(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); - return; } - long lastRemovedNotificationTs = partitioningRepository.dropPartitionsBefore(NOTIFICATION_TABLE_NAME, expTime, partitionDurationMs); - if (lastRemovedNotificationTs > 0) { - long gap = TimeUnit.MINUTES.toMillis(10); - long requestExpTime = lastRemovedNotificationTs - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; - int removed = notificationRequestDao.removeAllByCreatedTimeBefore(requestExpTime); - log.info("Removed {} outdated notification requests older than {}", removed, requestExpTime); + long gap = TimeUnit.MINUTES.toMillis(10); + long requestExpTime = expTime - TimeUnit.SECONDS.toMillis(NotificationRequestConfig.MAX_SENDING_DELAY) - gap; + cleanUpNotificationRequests(requestExpTime); + } + + private void cleanUpNotificationRequests(long expirationTime) { + log.info("Starting notification requests cleanup for records older than {}", Instant.ofEpochMilli(expirationTime)); + int totalRemoved = 0; + int tenantsProcessed = 0; + + // Clean up SYSADMIN's notification requests on the system node only + if (isSystemTenantPartitionMine()) { + try { + totalRemoved += cleanUpByTenant(TenantId.SYS_TENANT_ID, expirationTime); + } catch (Exception e) { + log.warn("Failed to clean up notification requests for sysadmin {}", TenantId.SYS_TENANT_ID, e); + } + } + // Each node cleans up notification requests for its own tenants + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + if (!isTenantPartitionMine(tenantId)) { + continue; + } + int tenantRemoved = cleanUpByTenant(tenantId, expirationTime); + totalRemoved += tenantRemoved; + tenantsProcessed++; + if (tenantRemoved > 0) { + log.trace("Removed {} notification requests for tenant {}", tenantRemoved, tenantId); + } + } catch (Exception e) { + log.warn("Failed to clean up notification requests for tenant {}", tenantId, e); + } } + + log.info("Notification requests cleanup completed. Processed {} tenants, removed {} total records older than {}", tenantsProcessed, totalRemoved, Instant.ofEpochMilli(expirationTime)); + } + + private int cleanUpByTenant(TenantId tenantId, long expirationTime) { + int totalRemoved = 0; + int batchRemoved; + + do { + batchRemoved = notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, removalBatchSize); + totalRemoved += batchRemoved; + + if (batchRemoved > 0) { + log.trace("Removed {} notification requests in batch for tenant {}", batchRemoved, tenantId); + } + } while (batchRemoved >= removalBatchSize); + + return totalRemoved; } } diff --git a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java index 9404a0ae72..a945bfd8a5 100644 --- a/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java +++ b/application/src/main/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpService.java @@ -15,69 +15,87 @@ */ package org.thingsboard.server.service.ttl.rpc; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.page.PageData; -import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.page.PageDataIterable; import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; -import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.dao.rpc.RpcDao; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantService; import org.thingsboard.server.queue.discovery.PartitionService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.ttl.AbstractCleanUpService; -import java.util.Date; +import java.time.Instant; import java.util.Optional; import java.util.concurrent.TimeUnit; -@TbCoreComponent -@Service @Slf4j -@RequiredArgsConstructor -public class RpcCleanUpService { - @Value("${sql.ttl.rpc.enabled}") - private boolean ttlTaskExecutionEnabled; +@Service +@TbCoreComponent +@ConditionalOnExpression("${sql.ttl.rpc.enabled:true}") +public class RpcCleanUpService extends AbstractCleanUpService { + + @Value("${sql.ttl.rpc.removal_batch_size:10000}") + private int removalBatchSize; + private final RpcDao rpcDao; private final TenantService tenantService; - private final PartitionService partitionService; private final TbTenantProfileCache tenantProfileCache; - private final RpcDao rpcDao; + + public RpcCleanUpService(TenantService tenantService, PartitionService partitionService, TbTenantProfileCache tenantProfileCache, RpcDao rpcDao) { + super(partitionService); + this.tenantService = tenantService; + this.tenantProfileCache = tenantProfileCache; + this.rpcDao = rpcDao; + } @Scheduled(initialDelayString = "#{T(org.apache.commons.lang3.RandomUtils).nextLong(0, ${sql.ttl.rpc.checking_interval})}", fixedDelayString = "${sql.ttl.rpc.checking_interval}") public void cleanUp() { - if (ttlTaskExecutionEnabled) { - PageLink tenantsBatchRequest = new PageLink(10_000, 0); - PageData tenantsIds; - do { - tenantsIds = tenantService.findTenantsIds(tenantsBatchRequest); - for (TenantId tenantId : tenantsIds.getData()) { - if (!partitionService.resolve(ServiceType.TB_CORE, tenantId, tenantId).isMyPartition()) { - continue; - } - - Optional tenantProfileConfiguration = tenantProfileCache.get(tenantId).getProfileConfiguration(); - if (tenantProfileConfiguration.isEmpty() || tenantProfileConfiguration.get().getRpcTtlDays() == 0) { - continue; - } - - long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays()); - long expirationTime = System.currentTimeMillis() - ttl; - - int totalRemoved = rpcDao.deleteOutdatedRpcByTenantId(tenantId, expirationTime); - - if (totalRemoved > 0) { - log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, new Date(expirationTime)); - } + PageDataIterable tenants = new PageDataIterable<>(tenantService::findTenantsIds, 10_000); + for (TenantId tenantId : tenants) { + try { + if (!isTenantPartitionMine(tenantId)) { + continue; } - tenantsBatchRequest = tenantsBatchRequest.nextPageLink(); - } while (tenantsIds.hasNext()); + Optional tenantProfileConfiguration = tenantProfileCache.get(tenantId).getProfileConfiguration(); + if (tenantProfileConfiguration.isEmpty() || tenantProfileConfiguration.get().getRpcTtlDays() == 0) { + continue; + } + + long ttl = TimeUnit.DAYS.toMillis(tenantProfileConfiguration.get().getRpcTtlDays()); + long expirationTime = System.currentTimeMillis() - ttl; + + int totalRemoved = cleanUpByTenant(tenantId, expirationTime); + + if (totalRemoved > 0) { + log.info("Removed {} outdated rpc(s) for tenant {} older than {}", totalRemoved, tenantId, Instant.ofEpochMilli(expirationTime)); + } + } catch (Exception e) { + log.warn("Failed to clean up rpc by ttl for tenant {}", tenantId, e); + } } } + private int cleanUpByTenant(TenantId tenantId, long expirationTime) { + int totalRemoved = 0; + int batchRemoved; + + do { + batchRemoved = rpcDao.deleteOutdatedRpcByTenantIdBatch(tenantId, expirationTime, removalBatchSize); + totalRemoved += batchRemoved; + + if (batchRemoved > 0) { + log.trace("Removed {} rpc in batch for tenant {}", batchRemoved, tenantId); + } + } while (batchRemoved >= removalBatchSize); + + return totalRemoved; + } + } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 42fd314563..d4e1a27ae7 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -173,6 +173,60 @@ security: path: "${SECURITY_JAVA_CACERTS_PATH:${java.home}/lib/security/cacerts}" # The password of the cacerts keystore file password: "${SECURITY_JAVA_CACERTS_PASSWORD:changeit}" + # HTTP security response headers configuration. + # These headers are set on responses from the ThingsBoard backend (tb-node). + # In microservice deployments, the web-ui (Express.js) has its own header configuration + # under msa/web-ui/config/ using the same environment variable names. + headers: + # X-Content-Type-Options header prevents browsers from MIME-sniffing the Content-Type. + # Safe to enable. Only disable if you intentionally serve resources with mismatched Content-Type. + x-content-type-options: + # Enable/disable X-Content-Type-Options header. Prevents browsers from MIME-sniffing the Content-Type + enabled: "${SECURITY_HEADERS_X_CONTENT_TYPE_OPTIONS_ENABLED:true}" + # Referrer-Policy header controls how much referrer info the browser sends with requests. + # The default 'strict-origin-when-cross-origin' matches the browser's built-in default, + # so enabling this does not change existing behavior — it just makes the policy explicit. + # Valid values: no-referrer, no-referrer-when-downgrade, origin, origin-when-cross-origin, + # same-origin, strict-origin, strict-origin-when-cross-origin, unsafe-url + referrer-policy: + # Enable/disable Referrer-Policy header + enabled: "${SECURITY_HEADERS_REFERRER_POLICY_ENABLED:true}" + # Referrer-Policy header value + value: "${SECURITY_HEADERS_REFERRER_POLICY_VALUE:strict-origin-when-cross-origin}" + # X-Frame-Options header protects against clickjacking attacks by preventing the page + # from being loaded in iframes on other domains. + # Disabled by default because ThingsBoard supports multi-domain deployments where + # the platform may be embedded in iframes on customer domains. + # WARNING: Enabling with DENY will block ALL iframe embedding including dashboards + # embedded on external sites. Use SAMEORIGIN to allow same-domain iframes only. + x-frame-options: + # Enable/disable X-Frame-Options header. Protects against clickjacking attacks + enabled: "${SECURITY_HEADERS_X_FRAME_OPTIONS_ENABLED:false}" + # Valid values: DENY, SAMEORIGIN + value: "${SECURITY_HEADERS_X_FRAME_OPTIONS_VALUE:SAMEORIGIN}" + # Content-Security-Policy header mitigates XSS and data injection attacks by restricting + # which resources the browser is allowed to load. + # Disabled by default because ThingsBoard supports multi-domain deployments and + # because custom HTML Card widgets may use inline scripts, inline styles, and + # external resources that a restrictive CSP would block. + # WARNING when enabling: A strict CSP (e.g. script-src 'self') will break: + # - HTML Card widgets with inline JavaScript + # - Custom widget types with inline scripts/styles + # - Widgets loading external resources (images, fonts, scripts) + # - Dashboard embedding via iframes (if frame-ancestors is restrictive) + # Use 'report-only: true' first to test the impact before enforcing. + # The default value covers core ThingsBoard functionality including OpenStreetMap, ArcGIS maps, + # solution template previews, WebSocket telemetry, and chart Web Workers. + # NOTE: Google Maps requires adding Google domains to script-src, which is not included by default. + # To add Google Maps support, append to script-src: https://maps.googleapis.com https://maps.gstatic.com + content-security-policy: + # Enable/disable Content-Security-Policy header. Mitigates XSS and data injection attacks + enabled: "${SECURITY_HEADERS_CONTENT_SECURITY_POLICY_ENABLED:false}" + # Full CSP directive string. The default value is a working policy for most ThingsBoard deployments + value: "${SECURITY_HEADERS_CONTENT_SECURITY_POLICY_VALUE:default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline'; img-src 'self' data: blob: https://img.thingsboard.io https://tile.openstreetmap.org https://*.tile.openstreetmap.org https://*.arcgis.com; font-src 'self' data:; connect-src 'self' ws: wss: https://*.arcgis.com https://*.arcgisonline.com; worker-src 'self' blob:; frame-ancestors 'self'}" + # If true, uses Content-Security-Policy-Report-Only header instead — the browser + # reports violations but does not enforce them. Use for testing before enforcing. + report-only: "${SECURITY_HEADERS_CONTENT_SECURITY_POLICY_REPORT_ONLY:false}" # Mail settings parameters mail: @@ -421,10 +475,11 @@ sql: edge_events_ttl: "${SQL_TTL_EDGE_EVENTS_TTL:2628000}" # Number of seconds. The current value corresponds to one month alarms: checking_interval: "${SQL_ALARMS_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours - removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # To delete outdated alarms not all at once but in batches + removal_batch_size: "${SQL_ALARMS_TTL_REMOVAL_BATCH_SIZE:3000}" # Batch size for records removal rpc: enabled: "${SQL_TTL_RPC_ENABLED:true}" # Enable/disable TTL (Time To Live) for rpc call records checking_interval: "${SQL_RPC_TTL_CHECKING_INTERVAL:7200000}" # Number of milliseconds. The current value corresponds to two hours + removal_batch_size: "${SQL_RPC_TTL_REMOVAL_BATCH_SIZE:10000}" # Batch size for records removal audit_logs: enabled: "${SQL_TTL_AUDIT_LOGS_ENABLED:true}" # Enable/disable TTL (Time To Live) for audit log records ttl: "${SQL_TTL_AUDIT_LOGS_SECS:0}" # Disabled by default. The accuracy of the cleanup depends on the sql.audit_logs.partition_size @@ -433,6 +488,7 @@ sql: enabled: "${SQL_TTL_NOTIFICATIONS_ENABLED:true}" # Enable/disable TTL (Time To Live) for notification center records ttl: "${SQL_TTL_NOTIFICATIONS_SECS:2592000}" # Default value - 30 days checking_interval_ms: "${SQL_TTL_NOTIFICATIONS_CHECKING_INTERVAL_MS:86400000}" # Default value - 1 day + removal_batch_size: "${SQL_TTL_NOTIFICATIONS_REMOVAL_BATCH_SIZE:10000}" # Batch size for records removal relations: max_level: "${SQL_RELATIONS_MAX_LEVEL:50}" # This value has to be reasonably small to prevent infinite recursion as early as possible pool_size: "${SQL_RELATIONS_POOL_SIZE:4}" # This value has to be reasonably small to prevent the relation query from blocking all other DB calls @@ -507,6 +563,10 @@ actors: # Comma-separated list of additional blocked destinations (IPs, CIDR subnets, or hostnames). # Example: "198.51.100.0/24,metadata.tencentyun.com,rancher-metadata" ssrf_additional_blocked_hosts: "${SSRF_ADDITIONAL_BLOCKED_HOSTS:}" + # Comma-separated list of allowed destinations that bypass SSRF blocking (IPs, CIDR subnets, or hostnames). + # Use this when your rule chains need to reach devices on private networks (e.g., 192.168.1.0/24). + # Example: "192.168.1.0/24,10.0.0.0/8,my-internal-service.corp" + ssrf_allowed_hosts: "${SSRF_ALLOWED_HOSTS:}" rpc: # Maximum number of persistent RPC call retries in case of failed request delivery. max_retries: "${ACTORS_RPC_MAX_RETRIES:5}" @@ -786,21 +846,28 @@ updates: # Enable/disable checks for the new version enabled: "${UPDATES_ENABLED:true}" -# Spring CORS configuration parameters +# Spring CORS configuration parameters. +# Controls the Access-Control-Allow-Origin and Access-Control-Allow-Credentials response headers. +# WARNING: The default configuration allows cross-origin requests from ANY domain with credentials. +# This means any website can make API requests on behalf of an authenticated user if the token +# is accessible (e.g., via XSS). For production deployments, restrict to your domain(s): +# TB_CORS_ALLOWED_ORIGIN_PATTERNS=https://your-domain.com +# For multi-domain deployments, list all allowed domains comma-separated: +# TB_CORS_ALLOWED_ORIGIN_PATTERNS=https://domain1.com,https://domain2.com spring.mvc.cors: mappings: # Intercept path "[/api/**]": #Comma-separated list of origins to allow. '*' allows all origins. When not set, CORS support is disabled. - allowed-origin-patterns: "*" + allowed-origin-patterns: "${TB_CORS_ALLOWED_ORIGIN_PATTERNS:*}" #Comma-separated list of methods to allow. '*' allows all methods. - allowed-methods: "*" + allowed-methods: "${TB_CORS_ALLOWED_METHODS:*}" #Comma-separated list of headers to allow in a request. '*' allows all headers. - allowed-headers: "*" + allowed-headers: "${TB_CORS_ALLOWED_HEADERS:*}" #How long, in seconds, the response from a pre-flight request can be cached by clients. - max-age: "1800" + max-age: "${TB_CORS_MAX_AGE:1800}" #Set whether credentials are supported. When not set, credentials are not supported. - allow-credentials: "true" + allow-credentials: "${TB_CORS_ALLOW_CREDENTIALS:true}" # General spring parameters spring.main.allow-circular-references: "true" # Spring Boot configuration property that controls whether circular dependencies between beans are allowed. @@ -1487,10 +1554,17 @@ edges: ssl: # Enable/disable SSL support enabled: "${EDGES_RPC_SSL_ENABLED:false}" - # Cert file to be used during TLS connectivity to the cloud + # Path to the server certificate file (holds server certificate or certificate chain, may include server private key). + # Accepts an absolute filesystem path (e.g. /etc/thingsboard/certChainFile.pem), + # a relative path resolved against the working directory first then the classpath, + # or a classpath resource with the explicit "classpath:" prefix (e.g. classpath:conf/certChainFile.pem). cert: "${EDGES_RPC_SSL_CERT:certChainFile.pem}" - # Private key file associated with the Cert certificate. This key is used in the encryption process during a secure connection + # Path to the server certificate private key file. Optional if the private key is already present in the cert file above. + # Supports the same path resolution as 'cert': absolute, relative/classpath, or "classpath:" prefix. + # Leave empty when using a combined PEM cert that already contains the private key. private_key: "${EDGES_RPC_SSL_PRIVATE_KEY:privateKeyFile.pem}" + # Server certificate private key password (optional). Leave empty if the key is not encrypted. + key_password: "${EDGES_RPC_SSL_KEY_PASSWORD:}" # Maximum size (in bytes) of inbound messages the cloud can handle from the edge. By default, it can handle messages up to 4 Megabytes max_inbound_message_size: "${EDGES_RPC_MAX_INBOUND_MESSAGE_SIZE:4194304}" # Maximum length of telemetry (time-series and attributes) message the cloud sends to the edge. By default, there is no limitation. diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 3b7286cf01..bbf3a3467e 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -405,6 +405,10 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { public void teardownWebTest() throws Exception { log.debug("Executing web test teardown"); + // Drain any pending housekeeper work left by the test body (e.g., bulk tenant deletes) + // before proceeding with teardown deletions, to avoid 90s per-tenant wait timing out. + awaitHousekeeperDrained(); + loginSysAdmin(); deleteTenant(tenantId); deleteDifferentTenant(); @@ -436,6 +440,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { .until(() -> storage.getLag("tb_housekeeper") == 0); } + protected void awaitHousekeeperDrained() { + Awaitility.await("housekeeper drained").atMost(5, TimeUnit.MINUTES).during(300, TimeUnit.MILLISECONDS) + .until(() -> storage.getLag("tb_housekeeper") == 0); + } + private List getAllTenants() throws Exception { List loadedTenants = new ArrayList<>(); PageLink pageLink = new PageLink(10); @@ -1052,7 +1061,7 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { assertThat(findRelationsByTo(entityTo)).hasSize(1); doDelete(urlDelete) - .andExpect(status().isInternalServerError()); + .andExpect(status().isBadRequest()); assertThat(findRelationsByTo(entityTo)).hasSize(1); } finally { diff --git a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java index 421f3785ca..b65d4178f0 100644 --- a/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/EntityViewControllerTest.java @@ -41,6 +41,7 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.DynamicPropertyRegistry; import org.springframework.test.context.DynamicPropertySource; import org.springframework.test.context.TestPropertySource; +import org.springframework.test.util.TestSocketUtils; import org.springframework.test.web.servlet.ResultActions; import org.thingsboard.common.util.ThingsBoardExecutors; import org.thingsboard.server.common.data.Customer; @@ -87,8 +88,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; import static org.thingsboard.server.dao.model.ModelConstants.NULL_UUID; -import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest.MQTT_PORT; -import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest.MQTT_URL; @TestPropertySource(properties = { "transport.mqtt.enabled=true", @@ -98,6 +97,15 @@ import static org.thingsboard.server.transport.mqtt.AbstractMqttIntegrationTest. @ContextConfiguration(classes = {EntityViewControllerTest.Config.class}) @DaoSqlTest public class EntityViewControllerTest extends AbstractControllerTest { + // Must NOT be imported from AbstractMqttIntegrationTest. That field is a static final initialized + // once per JVM. Other test classes (e.g. MqttGatewayRateLimitsTest, DeviceEdgeTest) share the same + // constant but produce a different Spring context cache key, so Spring creates a separate + // ApplicationContext for each of them. Every context starts its own MqttTransportService and tries + // to bind the same port, causing BindException when tests run in the same Surefire JVM fork. + // Declaring the port here gives this context its own independently allocated port. + static final int MQTT_PORT = TestSocketUtils.findAvailableTcpPort(); + static final String MQTT_URL = "tcp://localhost:" + MQTT_PORT; + @DynamicPropertySource static void props(DynamicPropertyRegistry registry) { log.warn("transport.mqtt.bind_port = {}", MQTT_PORT); diff --git a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java index 0f87dc1fd3..cc217b05d5 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TenantControllerTest.java @@ -43,6 +43,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.exception.TenantNotFoundException; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.TenantProfileId; import org.thingsboard.server.common.data.msg.TbMsgType; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; @@ -868,6 +869,26 @@ public class TenantControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService); } + @Test + public void testSaveTenantWithNonExistentTenantProfileId() throws Exception { + loginSysAdmin(); + Tenant tenant = new Tenant(); + tenant.setTitle("My tenant"); + tenant.setTenantProfileId(new TenantProfileId(UUID.randomUUID())); + + String responseBody = doPost("/api/tenant", tenant) + .andExpect(status().isBadRequest()) + .andReturn().getResponse().getContentAsString(); + + // Verify sanitized message format + assertThat(responseBody).contains("Constraint violation: fk_tenant_profile"); + // Verify raw SQL details are not returned + assertThat(responseBody).doesNotContain("could not execute statement"); + assertThat(responseBody).doesNotContain("insert or update on table"); + assertThat(responseBody).doesNotContain("tenant_profile_id"); + assertThat(responseBody).doesNotContain("is not present in table"); + } + private void testBroadcastEntityStateChangeEventNeverTenant() { Mockito.verify(tbClusterService, never()).onTenantChange(Mockito.any(Tenant.class), Mockito.isNull()); diff --git a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java index 87ba0ec3e8..6ec0f3ad64 100644 --- a/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/WebsocketApiTest.java @@ -19,13 +19,16 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; import lombok.extern.slf4j.Slf4j; import org.checkerframework.checker.nullness.qual.Nullable; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; @@ -60,7 +63,9 @@ import org.thingsboard.server.common.data.query.NumericFilterPredicate; import org.thingsboard.server.common.data.query.SingleEntityFilter; import org.thingsboard.server.common.data.query.TsValue; import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.dao.nosql.ResultSetSizeLimitExceededException; import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.dao.timeseries.TimeseriesService; import org.thingsboard.server.service.subscription.SubscriptionErrorCode; import org.thingsboard.server.service.subscription.TbAttributeSubscriptionScope; import org.thingsboard.server.service.telemetry.TelemetrySubscriptionService; @@ -95,6 +100,9 @@ public class WebsocketApiTest extends AbstractControllerTest { @Autowired private TelemetrySubscriptionService tsService; + @SpyBean + private TimeseriesService timeseriesService; + Device device; DeviceTypeFilter dtf; @@ -726,6 +734,41 @@ public class WebsocketApiTest extends AbstractControllerTest { Assert.assertNull(msg); } + @Test + public void testShouldSendWsUpdateMessageWhenTelemetryWasDeleted() throws Exception { + long now = System.currentTimeMillis() - 100; + TsKvEntry dataPoint = new BasicTsKvEntry(now, new LongDataEntry("temperature", 42L)); + List tsData = List.of(dataPoint); + sendTelemetry(device, tsData); + + List keys = List.of(new EntityKey(EntityKeyType.TIME_SERIES, "temperature")); + EntityDataUpdate update = getWsClient().subscribeLatestUpdate(keys, dtf); + + Assert.assertEquals(1, update.getCmdId()); + PageData pageData = update.getData(); + Assert.assertNotNull(pageData); + Assert.assertEquals(1, pageData.getData().size()); + Assert.assertEquals(device.getId(), pageData.getData().get(0).getEntityId()); + Assert.assertNotNull(pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature")); + Assert.assertEquals(now, pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getTs()); + Assert.assertEquals("42", pageData.getData().get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature").getValue()); + + // delete telemetry + getWsClient().registerWaitForUpdate(); + doDeleteAsync("/api/plugins/telemetry/DEVICE/" + device.getId() + "/timeseries/delete?keys=temperature&deleteAllDataForKeys=true", String.class); + update = getWsClient().parseDataReply(getWsClient().waitForUpdate()); + + Assert.assertEquals(1, update.getCmdId()); + + List listData = update.getUpdate(); + Assert.assertNotNull(listData); + Assert.assertEquals(1, listData.size()); + Assert.assertEquals(device.getId(), listData.get(0).getEntityId()); + Assert.assertNotNull(listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES)); + TsValue tsValue = listData.get(0).getLatest().get(EntityKeyType.TIME_SERIES).get("temperature"); + Assert.assertEquals(new TsValue(0, ""), tsValue); + } + @Test public void testEntityDataLatestAttrWsCmd() throws Exception { long now = System.currentTimeMillis(); @@ -965,6 +1008,34 @@ public class WebsocketApiTest extends AbstractControllerTest { } + @Test + public void testHistoryCmdSendsWsErrorOnResultSetSizeLimitExceeded() throws Exception { + ResultSetSizeLimitExceededException exception = new ResultSetSizeLimitExceededException(100L, 200L); + Mockito.doReturn(Futures.immediateFailedFuture(exception)) + .when(timeseriesService).findAllByQueries(Mockito.any(), Mockito.any(), Mockito.any()); + + List keys = List.of("temperature"); + long now = System.currentTimeMillis(); + + EntityDataUpdate errorUpdate = getWsClient().sendHistoryCmd(keys, now, TimeUnit.HOURS.toMillis(1), dtf); + assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); + assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); + } + + @Test + public void testTimeSeriesCmdSendsWsErrorOnResultSetSizeLimitExceeded() throws Exception { + ResultSetSizeLimitExceededException exception = new ResultSetSizeLimitExceededException(100L, 200L); + Mockito.doReturn(Futures.immediateFailedFuture(exception)) + .when(timeseriesService).findAllByQueries(Mockito.any(), Mockito.any(), Mockito.any()); + + List keys = List.of("temperature"); + long now = System.currentTimeMillis(); + + EntityDataUpdate errorUpdate = getWsClient().subscribeTsUpdate(keys, now, TimeUnit.HOURS.toMillis(1), dtf); + assertThat(errorUpdate.getErrorCode()).isEqualTo(SubscriptionErrorCode.INTERNAL_ERROR.getCode()); + assertThat(errorUpdate.getErrorMsg()).isEqualTo(exception.getMessage()); + } + private void sendTelemetry(Device device, List tsData) throws InterruptedException { CountDownLatch latch = new CountDownLatch(1); tsService.saveTimeseries(TimeseriesSaveRequest.builder() diff --git a/application/src/test/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSslTest.java b/application/src/test/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSslTest.java new file mode 100644 index 0000000000..04a1b87833 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/edge/rpc/EdgeGrpcSslTest.java @@ -0,0 +1,272 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.edge.rpc; + +import io.grpc.ManagedChannel; +import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.GrpcSslContexts; +import io.grpc.netty.shaded.io.grpc.netty.NettyChannelBuilder; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.openssl.jcajce.JcaPEMWriter; +import org.bouncycastle.openssl.jcajce.JcePEMEncryptorBuilder; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import org.bouncycastle.util.io.pem.PemObject; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.controller.AbstractWebTest; +import org.thingsboard.server.gen.edge.v1.EdgeRpcServiceGrpc; + +import java.io.ByteArrayInputStream; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.Security; +import java.security.cert.X509Certificate; +import java.security.spec.ECGenParameterSpec; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; + +/** + * Tests for Edge gRPC SSL setup using the production {@link EdgeGrpcService#setupSsl} method. + *

+ * Covers: + * 1. Separate cert and key PEM inputs + * 2. Combined PEM (cert + key in one file) + * 3. Encrypted private key with password + * 4. Missing key in combined PEM → error + *

+ * Each scenario is parameterized across key types: RSA-2048, RSA-4096, EC P-256, EC P-384. + */ +class EdgeGrpcSslTest { + + static { + if (Security.getProvider(BouncyCastleProvider.PROVIDER_NAME) == null) { + Security.addProvider(new BouncyCastleProvider()); + } + } + + enum KeyType { + RSA_2048("RSA", 2048, null, "SHA256withRSA"), + RSA_4096("RSA", 4096, null, "SHA256withRSA"), + EC_P256("EC", 256, "secp256r1", "SHA256withECDSA"), + EC_P384("EC", 384, "secp384r1", "SHA384withECDSA"); + + final String algorithm; + final int size; + final String curve; + final String sigAlg; + + KeyType(String algorithm, int size, String curve, String sigAlg) { + this.algorithm = algorithm; + this.size = size; + this.curve = curve; + this.sigAlg = sigAlg; + } + + KeyPair generateKeyPair() throws Exception { + KeyPairGenerator kpg = KeyPairGenerator.getInstance(algorithm); + if (curve != null) { + kpg.initialize(new ECGenParameterSpec(curve)); + } else { + kpg.initialize(size); + } + return kpg.generateKeyPair(); + } + } + + private final List tempFiles = new ArrayList<>(); + private Server server; + private ManagedChannel channel; + + @AfterEach + void cleanup() throws Exception { + if (channel != null) { + channel.shutdownNow().awaitTermination(2, TimeUnit.SECONDS); + } + if (server != null) { + server.shutdownNow().awaitTermination(2, TimeUnit.SECONDS); + } + for (Path p : tempFiles) { + Files.deleteIfExists(p); + } + } + + @ParameterizedTest(name = "separateCertAndKey_{0}") + @EnumSource(KeyType.class) + void separateCertAndKey(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + + Path certFile = writeTempPem("cert", cert); + Path keyFile = writeTempPem("key", kp.getPrivate()); + + server = startServer(certFile.toString(), keyFile.toString(), null); + assertTlsConnectivity(cert); + } + + @ParameterizedTest(name = "combinedPemWithCertAndKey_{0}") + @EnumSource(KeyType.class) + void combinedPemWithCertAndKey(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + + Path combinedFile = writeTempPem("combined", cert, kp.getPrivate()); + + server = startServer(combinedFile.toString(), "", null); + assertTlsConnectivity(cert); + } + + // RSA-only: BouncyCastle writes encrypted EC keys in traditional PEM format (BEGIN EC PRIVATE KEY), + // which after decryption produces a PEMKeyPair without public key info — causing PemSslCredentials + // to fail with "Cannot invoke SubjectPublicKeyInfo.getEncoded() because getPublicKeyInfo() is null". + @ParameterizedTest(name = "encryptedPrivateKey_{0}") + @EnumSource(value = KeyType.class, names = {"RSA_2048", "RSA_4096"}) + void encryptedPrivateKey(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + String password = "test-password"; + + Path combinedFile = writeTempPemEncrypted("enc-combined", password, cert, kp.getPrivate()); + + server = startServer(combinedFile.toString(), "", password); + assertTlsConnectivity(cert); + } + + @ParameterizedTest(name = "combinedPemWithCertOnly_throwsException_{0}") + @EnumSource(KeyType.class) + void combinedPemWithCertOnly_throwsException(KeyType keyType) throws Exception { + KeyPair kp = keyType.generateKeyPair(); + X509Certificate cert = generateSelfSignedCert(kp, keyType.sigAlg); + + Path certOnlyFile = writeTempPem("cert-only", cert); + + assertThatThrownBy(() -> startServer(certOnlyFile.toString(), "", null)) + .isInstanceOf(IllegalArgumentException.class); + } + + // --- Server startup using production EdgeGrpcService.setupSsl() --- + + private Server startServer(String certFileResource, String privateKeyResource, String keyPassword) throws Exception { + EdgeGrpcService edgeGrpcService = new EdgeGrpcService(); + ReflectionTestUtils.setField(edgeGrpcService, "certFileResource", certFileResource); + ReflectionTestUtils.setField(edgeGrpcService, "privateKeyResource", privateKeyResource); + ReflectionTestUtils.setField(edgeGrpcService, "keyPassword", keyPassword != null ? keyPassword : ""); + + NettyServerBuilder builder = NettyServerBuilder.forPort(0) + .addService(new EdgeRpcServiceGrpc.EdgeRpcServiceImplBase() {}); + + edgeGrpcService.setupSsl(builder); + + return builder.build().start(); + } + + private void assertTlsConnectivity(X509Certificate trustedCert) throws Exception { + String certPem = toPem(trustedCert); + var clientSsl = GrpcSslContexts.forClient() + .trustManager(new ByteArrayInputStream(certPem.getBytes(StandardCharsets.UTF_8))) + .build(); + + channel = NettyChannelBuilder.forAddress("localhost", server.getPort()) + .sslContext(clientSsl) + .build(); + + channel.getState(true); // trigger connection attempt + await().atMost(AbstractWebTest.TIMEOUT, TimeUnit.SECONDS) + .pollInterval(50, TimeUnit.MILLISECONDS) + .untilAsserted(() -> { + var state = channel.getState(false); + if (state == io.grpc.ConnectivityState.TRANSIENT_FAILURE) { + throw new AssertionError("TLS handshake failed: channel in TRANSIENT_FAILURE"); + } + assertThat(state).isEqualTo(io.grpc.ConnectivityState.READY); + }); + } + + // --- Cert/key generation --- + + private X509Certificate generateSelfSignedCert(KeyPair kp, String sigAlg) throws Exception { + X500Name subject = new X500Name("CN=localhost"); + Date now = new Date(); + return new JcaX509CertificateConverter().getCertificate( + new JcaX509v3CertificateBuilder( + subject, BigInteger.ONE, now, + new Date(now.getTime() + TimeUnit.DAYS.toMillis(1)), + subject, kp.getPublic()) + .build(new JcaContentSignerBuilder(sigAlg).build(kp.getPrivate()))); + } + + // --- PEM file helpers --- + + private String toPem(Object obj) throws Exception { + java.io.StringWriter sw = new java.io.StringWriter(); + try (JcaPEMWriter w = new JcaPEMWriter(sw)) { + w.writeObject(obj); + } + return sw.toString(); + } + + private Path writeTempPem(String prefix, Object... objects) throws Exception { + Path p = Files.createTempFile(prefix + "-", ".pem"); + tempFiles.add(p); + try (JcaPEMWriter w = new JcaPEMWriter(Files.newBufferedWriter(p))) { + for (Object o : objects) { + w.writeObject(toPkcs8IfKey(o)); + } + } + return p; + } + + private Path writeTempPemEncrypted(String prefix, String password, Object... objects) throws Exception { + Path p = Files.createTempFile(prefix + "-", ".pem"); + tempFiles.add(p); + var encryptor = new JcePEMEncryptorBuilder("AES-256-CBC") + .setProvider(BouncyCastleProvider.PROVIDER_NAME) + .build(password.toCharArray()); + try (JcaPEMWriter w = new JcaPEMWriter(Files.newBufferedWriter(p))) { + for (Object o : objects) { + if (o instanceof PrivateKey) { + w.writeObject(o, encryptor); + } else { + w.writeObject(o); + } + } + } + return p; + } + + private Object toPkcs8IfKey(Object o) { + if (o instanceof PrivateKey pk) { + return new PemObject("PRIVATE KEY", pk.getEncoded()); + } + return o; + } +} diff --git a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java index c54ffe4a07..05ab1165f6 100644 --- a/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/entitiy/EntityServiceTest.java @@ -115,6 +115,8 @@ import java.util.Map; import java.util.Random; import java.util.UUID; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -1749,13 +1751,13 @@ public class EntityServiceTest extends AbstractControllerTest { } @Test - public void testFindTenantTelemetry() { + public void testFindTenantTelemetry() throws ExecutionException, InterruptedException, TimeoutException { // save timeseries by sys admin BasicTsKvEntry timeseries = new BasicTsKvEntry(42L, new DoubleDataEntry("temperature", 45.5)); - timeseriesService.save(TenantId.SYS_TENANT_ID, tenantId, timeseries); + timeseriesService.save(TenantId.SYS_TENANT_ID, tenantId, timeseries).get(TIMEOUT, TimeUnit.SECONDS); AttributeKvEntry attr = new BaseAttributeKvEntry(new LongDataEntry("attr", 10L), 42L); - attributesService.save(TenantId.SYS_TENANT_ID, tenantId, SERVER_SCOPE, List.of(attr)); + attributesService.save(TenantId.SYS_TENANT_ID, tenantId, SERVER_SCOPE, List.of(attr)).get(TIMEOUT, TimeUnit.SECONDS); SingleEntityFilter singleEntityFilter = new SingleEntityFilter(); singleEntityFilter.setSingleEntity(AliasEntityId.fromEntityId(tenantId)); diff --git a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java index cf37afad5c..0c12767b60 100644 --- a/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/housekeeper/HousekeeperServiceTest.java @@ -23,8 +23,8 @@ import org.junit.Test; import org.mockito.ArgumentMatcher; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.metadata.TbGetAttributesNode; @@ -127,10 +127,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. }) public class HousekeeperServiceTest extends AbstractControllerTest { - @SpyBean + @MockitoSpyBean private HousekeeperService housekeeperService; - @SpyBean + @MockitoSpyBean private HousekeeperReprocessingService housekeeperReprocessingService; + @MockitoSpyBean + private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; @Autowired private EventService eventService; @Autowired @@ -153,8 +155,6 @@ public class HousekeeperServiceTest extends AbstractControllerTest { private CustomerService customerService; @Autowired private DashboardService dashboardService; - @SpyBean - private TsHistoryDeletionTaskProcessor tsHistoryDeletionTaskProcessor; private TenantId tenantId; diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java new file mode 100644 index 0000000000..d74652325a --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/NotificationsCleanUpServiceTest.java @@ -0,0 +1,144 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.notification.NotificationRequestDao; +import org.thingsboard.server.dao.sqlts.insert.sql.SqlPartitioningRepository; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.PartitionService; + +import java.util.List; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class NotificationsCleanUpServiceTest { + + @Mock + private PartitionService partitionService; + @Mock + private SqlPartitioningRepository partitioningRepository; + @Mock + private NotificationRequestDao notificationRequestDao; + @Mock + private TenantService tenantService; + + private NotificationsCleanUpService cleanUpService; + + private static final int BATCH_SIZE = 3; + + @BeforeEach + public void setUp() { + cleanUpService = new NotificationsCleanUpService(partitionService, partitioningRepository, notificationRequestDao, tenantService); + ReflectionTestUtils.setField(cleanUpService, "ttlInSec", 2592000L); + ReflectionTestUtils.setField(cleanUpService, "partitionSizeInHours", 168); + ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); + } + + @Test + public void testBatchLoopCallsDaoMultipleTimes() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(System.currentTimeMillis()); + + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); + + // Sysadmin: returns 3 (full batch), then 1 (partial) -> 2 calls + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(1); + // Tenant: returns 3, 3, 0 -> 3 calls + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(BATCH_SIZE) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao, times(2)) + .removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE)); + verify(notificationRequestDao, times(3)) + .removeByTenantIdAndCreatedTimeBeforeBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); + } + + @Test + public void testSkipsTenantNotOnMyPartition() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); + when(partitionService.resolve(any(), eq(TenantId.SYS_TENANT_ID), eq(TenantId.SYS_TENANT_ID))) + .thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(System.currentTimeMillis()); + + // Sysadmin: no records + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); + TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); + when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); + when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); + + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao).removeByTenantIdAndCreatedTimeBeforeBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); + verify(notificationRequestDao, never()).removeByTenantIdAndCreatedTimeBeforeBatch(eq(otherTenant), anyLong(), anyInt()); + } + + @Test + public void testNoPartitionsDropped_stillCleansUpRequests() { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), any(), any())).thenReturn(myPartition); + when(partitioningRepository.dropPartitionsBefore(anyString(), anyLong(), anyLong())) + .thenReturn(0L); + + when(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(), 0, 0, false)); + + cleanUpService.cleanUp(); + + verify(notificationRequestDao).removeByTenantIdAndCreatedTimeBeforeBatch(eq(TenantId.SYS_TENANT_ID), anyLong(), eq(BATCH_SIZE)); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java b/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java new file mode 100644 index 0000000000..22cc1be1dd --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/ttl/rpc/RpcCleanUpServiceTest.java @@ -0,0 +1,136 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.ttl.rpc; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.test.util.ReflectionTestUtils; +import org.thingsboard.server.common.data.TenantProfile; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileConfiguration; +import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; +import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; +import org.thingsboard.server.dao.rpc.RpcDao; +import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import org.thingsboard.server.dao.tenant.TenantService; +import org.thingsboard.server.queue.discovery.PartitionService; + +import java.util.List; +import java.util.UUID; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +public class RpcCleanUpServiceTest { + + @Mock + private PartitionService partitionService; + @Mock + private RpcDao rpcDao; + @Mock + private TenantService tenantService; + @Mock + private TbTenantProfileCache tenantProfileCache; + + private RpcCleanUpService cleanUpService; + + private static final int BATCH_SIZE = 3; + + @BeforeEach + public void setUp() { + cleanUpService = new RpcCleanUpService(tenantService, partitionService, tenantProfileCache, rpcDao); + ReflectionTestUtils.setField(cleanUpService, "removalBatchSize", BATCH_SIZE); + } + + @Test + public void testBatchLoopCallsDaoMultipleTimes() { + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + setupTenant(tenantId, 7); + + // Returns 3 (full batch), 3 (full batch), 1 (partial) -> 3 calls + when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE))) + .thenReturn(BATCH_SIZE) + .thenReturn(BATCH_SIZE) + .thenReturn(1); + + cleanUpService.cleanUp(); + + verify(rpcDao, times(3)).deleteOutdatedRpcByTenantIdBatch(eq(tenantId), anyLong(), eq(BATCH_SIZE)); + } + + @Test + public void testSkipsTenantNotOnMyPartition() { + TenantId myTenant = TenantId.fromUUID(UUID.randomUUID()); + TenantId otherTenant = TenantId.fromUUID(UUID.randomUUID()); + + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + TopicPartitionInfo notMyPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(false).build(); + + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(myTenant, otherTenant), 2, 1, false)); + when(partitionService.resolve(any(), eq(myTenant), eq(myTenant))).thenReturn(myPartition); + when(partitionService.resolve(any(), eq(otherTenant), eq(otherTenant))).thenReturn(notMyPartition); + + setupTenantProfile(myTenant, 7); + when(rpcDao.deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE))) + .thenReturn(0); + + cleanUpService.cleanUp(); + + verify(rpcDao).deleteOutdatedRpcByTenantIdBatch(eq(myTenant), anyLong(), eq(BATCH_SIZE)); + verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(eq(otherTenant), anyLong(), anyInt()); + } + + @Test + public void testSkipsTenantWithZeroTtl() { + TenantId tenantId = TenantId.fromUUID(UUID.randomUUID()); + setupTenant(tenantId, 0); + + cleanUpService.cleanUp(); + + verify(rpcDao, never()).deleteOutdatedRpcByTenantIdBatch(any(), anyLong(), anyInt()); + } + + private void setupTenant(TenantId tenantId, int rpcTtlDays) { + TopicPartitionInfo myPartition = TopicPartitionInfo.builder().topic("tb_core").myPartition(true).build(); + when(partitionService.resolve(any(), eq(tenantId), eq(tenantId))).thenReturn(myPartition); + when(tenantService.findTenantsIds(any())) + .thenReturn(new PageData<>(List.of(tenantId), 1, 1, false)); + setupTenantProfile(tenantId, rpcTtlDays); + } + + private void setupTenantProfile(TenantId tenantId, int rpcTtlDays) { + TenantProfile profile = new TenantProfile(); + TenantProfileData profileData = new TenantProfileData(); + DefaultTenantProfileConfiguration config = new DefaultTenantProfileConfiguration(); + config.setRpcTtlDays(rpcTtlDays); + profileData.setConfiguration(config); + profile.setProfileData(profileData); + when(tenantProfileCache.get(tenantId)).thenReturn(profile); + } + +} diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java index 9bed9bc483..35e12691c9 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/client/FwLwM2MDevice.java @@ -171,7 +171,14 @@ public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { if (this.leshanClient != null) { log.info("Stop/reboot LwM2M client {}", this.leshanClient.getEndpoint(identity)); - this.leshanClient.stop(false); + try { + this.leshanClient.stop(false); + } catch (Exception stopEx) { + // Leshan may throw NPE during CoAP observe-relation cleanup when the server + // reference is null (race condition in NotificationDataStore.toKey()). + // The client is still considered stopped at this point — proceed with restart. + log.warn("Exception during LwM2M client stop, proceeding with restart: {}", stopEx.getMessage()); + } log.info("Start after update fw LwM2M client {}", this.leshanClient.getEndpoint(identity)); this.leshanClient.start(); @@ -193,7 +200,7 @@ public class FwLwM2MDevice extends BaseInstanceEnabler implements Destroyable { } catch (Exception e) { log.error("Error during firmware update", e); } - }, 0, TimeUnit.SECONDS); // start immediately, without further delay + }, 1, TimeUnit.SECONDS); // delay 1 sec to allow CoAP Execute response to be delivered before client stops } protected void setLeshanClient(LeshanClient leshanClient) { diff --git a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java index 432fac6d1a..c61ca0b123 100644 --- a/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/lwm2m/rpc/sql/RpcLwm2mIntegrationReadCollectedValueTest.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import org.junit.Before; import org.junit.Test; +import org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper; import org.thingsboard.server.transport.lwm2m.rpc.AbstractRpcLwM2MIntegrationTest; import java.util.concurrent.atomic.AtomicReference; import static java.util.concurrent.TimeUnit.SECONDS; @@ -37,6 +38,12 @@ import static org.thingsboard.server.transport.lwm2m.Lwm2mTestHelper.RESOURCE_ID @Slf4j public class RpcLwm2mIntegrationReadCollectedValueTest extends AbstractRpcLwM2MIntegrationTest { + @Before + public void resetCollectedValueTimestamps() { + Lwm2mTestHelper.RESOURCE_ID_3303_12_5700_TS_0 = 0; + Lwm2mTestHelper.RESOURCE_ID_3303_12_5700_TS_1 = 0; + } + /** * Read {"id":"/3303/12/5700"} * Trigger a Send operation from the client with multiple values for the same resource as a payload diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index a7307b2308..5f43aa7e00 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -420,6 +420,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt Awaitility.await() .atMost(10, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() { }); diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java index 4e4d0debf9..a9bdbb55db 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/AbstractMqttV5ClientSparkplugTest.java @@ -191,6 +191,7 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte AtomicReference device = new AtomicReference<>(); await(alias + "find device [" + deviceName + "] after created") .atMost(200, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { device.set(doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class)); return device.get() != null; @@ -236,6 +237,7 @@ public abstract class AbstractMqttV5ClientSparkplugTest extends AbstractMqttInte AtomicReference device = new AtomicReference<>(); await(alias + "find device [" + deviceName + "] after created") .atMost(200, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { device.set(doGet("/api/tenant/devices?deviceName=" + deviceName, Device.class)); return device.get() != null; diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java index 756c8e603c..d42adfbcf0 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/sparkplug/attributes/AbstractMqttV5ClientSparkplugAttributesTest.java @@ -468,6 +468,7 @@ public abstract class AbstractMqttV5ClientSparkplugAttributesTest extends Abstra AtomicReference> actualKeys = new AtomicReference<>(); await(alias + SparkplugMessageType.NBIRTH.name()) .atMost(40, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { actualKeys.set(doGetAsyncTyped(urlTemplate, new TypeReference<>() { })); @@ -483,6 +484,7 @@ public abstract class AbstractMqttV5ClientSparkplugAttributesTest extends Abstra AtomicReference> actualKeys = new AtomicReference<>(); await(alias + SparkplugMessageType.DBIRTH.name()) .atMost(40, TimeUnit.SECONDS) + .ignoreExceptions() .until(() -> { actualKeys.set(doGetAsyncTyped(urlTemplate, new TypeReference<>() { })); diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index e79289340c..7f0ab964d6 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -44,6 +44,7 @@ queue.transport_api.response_poll_interval=5 queue.transport.poll_interval=5 queue.core.poll-interval=5 queue.core.partitions=2 +queue.core.housekeeper.task-reprocessing-delay-ms=0 queue.rule-engine.poll-interval=5 queue.rule-engine.stats.enabled=true diff --git a/build.sh b/build.sh index d2a965f281..0342aa9cbf 100755 --- a/build.sh +++ b/build.sh @@ -31,7 +31,7 @@ java -version #echo "Cleaning ui-ngx/node_modules" && rm -rf ui-ngx/node_modules MAVEN_OPTS="-Xmx1024m" NODE_OPTIONS="--max_old_space_size=4096" DOCKER_CLI_EXPERIMENTAL=enabled DOCKER_BUILDKIT=0 \ -mvn -T2 license:format clean install -DskipTests \ +mvn -T6 license:format clean install -DskipTests -Dpkg.skip=true \ $PROJECTS --also-make # \ # -Dpush-docker-amd-arm-images diff --git a/common/actor/pom.xml b/common/actor/pom.xml index b6084940e2..e4d566a779 100644 --- a/common/actor/pom.xml +++ b/common/actor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/cache/pom.xml b/common/cache/pom.xml index 9b446af44c..4d4ece7c83 100644 --- a/common/cache/pom.xml +++ b/common/cache/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/cluster-api/pom.xml b/common/cluster-api/pom.xml index ff2909e89b..4fcde696b2 100644 --- a/common/cluster-api/pom.xml +++ b/common/cluster-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/coap-server/pom.xml b/common/coap-server/pom.xml index b35d42ad4c..d3d62ddade 100644 --- a/common/coap-server/pom.xml +++ b/common/coap-server/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/dao-api/pom.xml b/common/dao-api/pom.xml index 938943f6a7..43f441b5a2 100644 --- a/common/dao-api/pom.xml +++ b/common/dao-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/data/pom.xml b/common/data/pom.xml index 69c7abd93b..9e1d24ef26 100644 --- a/common/data/pom.xml +++ b/common/data/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java index dea590295e..d66ec046de 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsDeletionHousekeeperTask.java @@ -23,6 +23,7 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -32,6 +33,9 @@ import java.util.UUID; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9214680001573764374L; + private List alarms; public AlarmsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java index 0313190056..445850d387 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/AlarmsUnassignHousekeeperTask.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -33,6 +34,9 @@ import java.util.UUID; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class AlarmsUnassignHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9156667024462937756L; + private String userTitle; private List alarms; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java index fe25a98a1d..c1a233b1c4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/EntitiesDeletionHousekeeperTask.java @@ -23,6 +23,7 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.util.List; import java.util.UUID; @@ -32,6 +33,9 @@ import java.util.UUID; @NoArgsConstructor public class EntitiesDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 9009068831061529286L; + private EntityType entityType; private List entities; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java index 875ef2765f..2df7cf4dd4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/HousekeeperTask.java @@ -29,6 +29,7 @@ import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; import java.io.Serializable; @JsonIgnoreProperties(ignoreUnknown = true) @@ -45,6 +46,9 @@ import java.io.Serializable; @NoArgsConstructor(access = AccessLevel.PROTECTED) public class HousekeeperTask implements Serializable { + @Serial + private static final long serialVersionUID = -2585974110832225152L; + private TenantId tenantId; private EntityId entityId; private HousekeeperTaskType taskType; diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java index cd3e94e5c6..931c2931cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/LatestTsDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class LatestTsDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 5193191938513490138L; + private String key; public LatestTsDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java index 443d929d8b..be7ff6f7ec 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TenantEntitiesDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor public class TenantEntitiesDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = -8033108795318393447L; + private EntityType entityType; public TenantEntitiesDeletionHousekeeperTask(TenantId tenantId, EntityType entityType) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java index d9315f0ff4..b520899ca4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/housekeeper/TsHistoryDeletionHousekeeperTask.java @@ -23,12 +23,17 @@ import lombok.ToString; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; +import java.io.Serial; + @Data @ToString(callSuper = true) @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) public class TsHistoryDeletionHousekeeperTask extends HousekeeperTask { + @Serial + private static final long serialVersionUID = 4573851542705079043L; + private String key; public TsHistoryDeletionHousekeeperTask(TenantId tenantId, EntityId entityId, String key) { diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java index cb4092f433..595e1aa26b 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/kv/TsKvEntry.java @@ -37,4 +37,9 @@ public interface TsKvEntry extends KvEntry, HasVersion { return new TsValue(getTs(), getValueAsString()); } + @JsonIgnore + default boolean isDeletedEntry() { + return getTs() == 0 && (getValue() == null || getValueAsString().isEmpty()); + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java index fda73b8059..de4e05a2cb 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/NotificationRuleRecipientsConfig.java @@ -17,6 +17,7 @@ package org.thingsboard.server.common.data.notification.rule; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonSubTypes.Type; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -29,7 +30,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; -@JsonIgnoreProperties +@JsonIgnoreProperties(ignoreUnknown = true) @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "triggerType", visible = true, include = JsonTypeInfo.As.EXISTING_PROPERTY, defaultImpl = DefaultNotificationRuleRecipientsConfig.class) @JsonSubTypes({ @Type(name = "ALARM", value = EscalatedNotificationRuleRecipientsConfig.class), @@ -38,6 +39,7 @@ import java.util.UUID; public abstract class NotificationRuleRecipientsConfig implements Serializable { @NotNull + @JsonProperty("triggerType") private NotificationRuleTriggerType triggerType; @JsonIgnore diff --git a/common/discovery-api/pom.xml b/common/discovery-api/pom.xml index ac66b5b6cf..3df2e3f204 100644 --- a/common/discovery-api/pom.xml +++ b/common/discovery-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/edge-api/pom.xml b/common/edge-api/pom.xml index 6bea266508..1310f2cd69 100644 --- a/common/edge-api/pom.xml +++ b/common/edge-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/edge-api/src/main/proto/edge.proto b/common/edge-api/src/main/proto/edge.proto index 79b69e60c8..e2f565c7a3 100644 --- a/common/edge-api/src/main/proto/edge.proto +++ b/common/edge-api/src/main/proto/edge.proto @@ -46,6 +46,7 @@ enum EdgeVersion { V_4_2_0 = 12; V_4_2_1_2 = 14; V_4_2_2 = 4220; + V_4_2_2_1 = 4221; V_LATEST = 99999; } diff --git a/common/edqs/pom.xml b/common/edqs/pom.xml index 97977ca33a..56d2f2cb0c 100644 --- a/common/edqs/pom.xml +++ b/common/edqs/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/message/pom.xml b/common/message/pom.xml index ade5a82988..6221bd74b3 100644 --- a/common/message/pom.xml +++ b/common/message/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/pom.xml b/common/pom.xml index 8011921449..f0e6936ec8 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard common diff --git a/common/proto/pom.xml b/common/proto/pom.xml index cefdf52c51..dbf6a62211 100644 --- a/common/proto/pom.xml +++ b/common/proto/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/queue/pom.xml b/common/queue/pom.xml index d0a3d26138..0c4364c790 100644 --- a/common/queue/pom.xml +++ b/common/queue/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/script/pom.xml b/common/script/pom.xml index 73d24e3c34..16793dc718 100644 --- a/common/script/pom.xml +++ b/common/script/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/script/remote-js-client/pom.xml b/common/script/remote-js-client/pom.xml index a875b5bbc6..5aa9ddbc4f 100644 --- a/common/script/remote-js-client/pom.xml +++ b/common/script/remote-js-client/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT script org.thingsboard.common.script diff --git a/common/script/script-api/pom.xml b/common/script/script-api/pom.xml index 2a4941b117..bc4c9192c5 100644 --- a/common/script/script-api/pom.xml +++ b/common/script/script-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT script org.thingsboard.common.script diff --git a/common/stats/pom.xml b/common/stats/pom.xml index 61a1c9a912..84a4dceee3 100644 --- a/common/stats/pom.xml +++ b/common/stats/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/coap/pom.xml b/common/transport/coap/pom.xml index f1a42037b6..42170bdad5 100644 --- a/common/transport/coap/pom.xml +++ b/common/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/http/pom.xml b/common/transport/http/pom.xml index 11230ec7e8..e832433cb3 100644 --- a/common/transport/http/pom.xml +++ b/common/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/lwm2m/pom.xml b/common/transport/lwm2m/pom.xml index 4b7f7262b3..b1594c1c3d 100644 --- a/common/transport/lwm2m/pom.xml +++ b/common/transport/lwm2m/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java index 4b2ef07994..8158e82a81 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbLwM2mRedisRegistrationStore.java @@ -323,22 +323,24 @@ public class TbLwM2mRedisRegistrationStore implements RegistrationStore, Startab @Override public Iterator getAllRegistrations() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { Collection list = new LinkedList<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(REG_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); - list.add(deserializeReg(element)); + byte[] element = getConnection.get(key); + if (element != null) { + list.add(deserializeReg(element)); + } }); }); return list.iterator(); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java index 3293cd8b53..4beefb4896 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStore.java @@ -61,21 +61,21 @@ public class TbRedisLwM2MClientStore implements TbLwM2MClientStore { @Override public Set getAll() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { Set clients = new HashSet<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(CLIENT_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); + byte[] element = getConnection.get(key); if (element != null) { try { clients.add(deserialize(element)); diff --git a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java index 73b6f3c8df..31a78234d0 100644 --- a/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java +++ b/common/transport/lwm2m/src/main/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MModelConfigStore.java @@ -35,22 +35,24 @@ public class TbRedisLwM2MModelConfigStore implements TbLwM2MModelConfigStore { @Override public List getAll() { - try (var connection = connectionFactory.getConnection()) { + try (var scanConnection = connectionFactory.getConnection(); + var getConnection = connectionFactory.getConnection()) { List configs = new ArrayList<>(); ScanOptions scanOptions = ScanOptions.scanOptions().count(100).match(MODEL_EP + "*").build(); List> scans = new ArrayList<>(); - if (connection instanceof RedisClusterConnection) { - ((RedisClusterConnection) connection).clusterGetNodes().forEach(node -> { - scans.add(((RedisClusterConnection) connection).scan(node, scanOptions)); - }); + if (scanConnection instanceof RedisClusterConnection clusterConnection) { + clusterConnection.clusterGetNodes().forEach(node -> + scans.add(clusterConnection.scan(node, scanOptions))); } else { - scans.add(connection.scan(scanOptions)); + scans.add(scanConnection.scan(scanOptions)); } scans.forEach(scan -> { scan.forEachRemaining(key -> { - byte[] element = connection.get(key); - configs.add(JacksonUtil.fromBytes(element, LwM2MModelConfig.class)); + byte[] element = getConnection.get(key); + if (element != null) { + configs.add(JacksonUtil.fromBytes(element, LwM2MModelConfig.class)); + } }); }); return configs; diff --git a/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java b/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java new file mode 100644 index 0000000000..9fe29c1188 --- /dev/null +++ b/common/transport/lwm2m/src/test/java/org/thingsboard/server/transport/lwm2m/server/store/TbRedisLwM2MClientStoreTest.java @@ -0,0 +1,137 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.transport.lwm2m.server.store; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.ScanOptions; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2MClientState; +import org.thingsboard.server.transport.lwm2m.server.client.LwM2mClient; + +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.thingsboard.server.transport.lwm2m.server.store.util.LwM2MClientSerDes.serialize; + +/** + * Verifies that {@link TbRedisLwM2MClientStore#getAll()} uses separate connections for + * SCAN and GET operations to prevent Jedis 5.x response-ordering corruption that occurs + * when both commands share the same connection. + */ +@ExtendWith(MockitoExtension.class) +class TbRedisLwM2MClientStoreTest { + + @Mock + RedisConnectionFactory connectionFactory; + + @Mock + RedisConnection scanConnection; + + @Mock + RedisConnection getConnection; + + TbRedisLwM2MClientStore store; + + @BeforeEach + void setUp() { + // First getConnection() call → scanConnection, second → getConnection + when(connectionFactory.getConnection()) + .thenReturn(scanConnection) + .thenReturn(getConnection); + store = new TbRedisLwM2MClientStore(connectionFactory); + } + + @Test + void getAll_returnsSingleClient() { + LwM2mClient client = new LwM2mClient("nodeId", "testEndpoint"); + client.setState(LwM2MClientState.REGISTERED); + byte[] key = "CLIENT#EP#testEndpoint".getBytes(); + byte[] value = serialize(client); + + // Cursor created before thenReturn to avoid Mockito unfinished-stubbing error + Cursor cursor = cursorOf(key); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + when(getConnection.get(key)).thenReturn(value); + + Set result = store.getAll(); + + assertThat(result).hasSize(1); + assertThat(result.iterator().next().getEndpoint()).isEqualTo("testEndpoint"); + } + + @Test + void getAll_getIsNeverCalledOnScanConnection() { + Cursor cursor = cursorOf(); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + + store.getAll(); + + verify(scanConnection, never()).get(any(byte[].class)); + } + + @Test + void getAll_scanIsNeverCalledOnGetConnection() { + Cursor cursor = cursorOf(); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + + store.getAll(); + + verify(getConnection, never()).scan(any(ScanOptions.class)); + } + + @Test + void getAll_skipsKeyWhenValueIsNull() { + byte[] key = "CLIENT#EP#gone".getBytes(); + Cursor cursor = cursorOf(key); + when(scanConnection.scan(any(ScanOptions.class))).thenReturn(cursor); + // getConnection.get(key) returns null by default — no stubbing needed + + Set result = store.getAll(); + + assertThat(result).isEmpty(); + } + + /** + * Creates a mock {@link Cursor} that iterates over the given keys via {@code forEachRemaining}. + * The cursor is created separately (not inside a {@code thenReturn()} argument) to avoid + * Mockito's "unfinished stubbing" error caused by nested {@code when()} calls. + */ + @SuppressWarnings("unchecked") + private static Cursor cursorOf(byte[]... keys) { + Cursor cursor = mock(Cursor.class); + List keyList = List.of(keys); + doAnswer(inv -> { + Consumer action = inv.getArgument(0); + keyList.forEach(action); + return null; + }).when(cursor).forEachRemaining(any(Consumer.class)); + return cursor; + } +} diff --git a/common/transport/mqtt/pom.xml b/common/transport/mqtt/pom.xml index 4d5854eb15..5b947c806f 100644 --- a/common/transport/mqtt/pom.xml +++ b/common/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/pom.xml b/common/transport/pom.xml index 34dea5a354..ac89c33953 100644 --- a/common/transport/pom.xml +++ b/common/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/transport/snmp/pom.xml b/common/transport/snmp/pom.xml index 43a3e5cf2e..5c711fdc90 100644 --- a/common/transport/snmp/pom.xml +++ b/common/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport diff --git a/common/transport/transport-api/pom.xml b/common/transport/transport-api/pom.xml index 41375882eb..8fda8f4daf 100644 --- a/common/transport/transport-api/pom.xml +++ b/common/transport/transport-api/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.common - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.common.transport diff --git a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java index e5f81dafc4..34cc1151c4 100644 --- a/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java +++ b/common/transport/transport-api/src/main/java/org/thingsboard/server/common/transport/config/ssl/SslCredentialsWebServerCustomizer.java @@ -30,6 +30,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; import java.util.List; +import java.util.function.BiConsumer; import java.util.function.Consumer; @Component @@ -88,6 +89,11 @@ public class SslCredentialsWebServerCustomizer implements WebServerFactoryCustom public void addBundleUpdateHandler(String name, Consumer handler) { // no-op } + + @Override + public void addBundleRegisterHandler(BiConsumer handler) { + // no-op + } }; } diff --git a/common/util/pom.xml b/common/util/pom.xml index fcd87ea1db..f529772166 100644 --- a/common/util/pom.xml +++ b/common/util/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/common/util/src/main/java/org/thingsboard/common/util/SsrfProtectionValidator.java b/common/util/src/main/java/org/thingsboard/common/util/SsrfProtectionValidator.java index 15da77f663..9132d117ae 100644 --- a/common/util/src/main/java/org/thingsboard/common/util/SsrfProtectionValidator.java +++ b/common/util/src/main/java/org/thingsboard/common/util/SsrfProtectionValidator.java @@ -38,6 +38,7 @@ public class SsrfProtectionValidator { private static final Set BLOCKED_HOSTNAME_SUFFIXES = Set.of(".internal", ".local"); private static volatile AdditionalBlockedHosts additionalBlocked = AdditionalBlockedHosts.EMPTY; + private static volatile AllowedHosts allowedHosts = AllowedHosts.EMPTY; // Well-known cloud metadata endpoints not covered by the JDK checks (isLoopback, isSiteLocal, isLinkLocal) private static final List CLOUD_METADATA_RANGES = List.of( @@ -66,6 +67,13 @@ public class SsrfProtectionValidator { } String hostLower = host.toLowerCase(); + + // Allow-listed hostnames bypass all hostname and IP checks + AllowedHosts currentAllowed = allowedHosts; + if (currentAllowed.hostnames.contains(hostLower)) { + return; + } + if (BLOCKED_HOSTNAMES.contains(hostLower) || additionalBlocked.hostnames.contains(hostLower)) { throwBlockedHost(host); } @@ -98,7 +106,15 @@ public class SsrfProtectionValidator { } } - private static boolean isBlockedAddress(InetAddress address) { + public static boolean isBlockedAddress(InetAddress address) { + // Check allow-list first: allowed addresses bypass all block checks + AllowedHosts currentAllowed = allowedHosts; + for (CidrRange cidr : currentAllowed.cidrRanges) { + if (cidr.contains(address)) { + return false; + } + } + // Covers 127.0.0.0/8 and ::1 if (address.isLoopbackAddress()) { return true; @@ -142,14 +158,37 @@ public class SsrfProtectionValidator { throw new RuntimeException("URI is invalid: host '" + host + "' is not allowed"); } + public static boolean isEnabled() { + return enabled; + } + public static void setEnabled(boolean enabled) { SsrfProtectionValidator.enabled = enabled; } public static void setAdditionalBlockedHosts(List entries) { + ParsedHostEntries parsed = parseHostEntries(entries); + additionalBlocked = new AdditionalBlockedHosts(parsed.cidrRanges, parsed.hostnames); + if (!parsed.cidrRanges.isEmpty() || !parsed.hostnames.isEmpty()) { + log.info("SSRF additional blocked hosts configured: {} CIDR range(s), {} hostname(s)", parsed.cidrRanges.size(), parsed.hostnames.size()); + } + } + + public static void setAllowedHosts(List entries) { + ParsedHostEntries parsed = parseHostEntries(entries); + allowedHosts = new AllowedHosts(parsed.cidrRanges, parsed.hostnames); + if (!parsed.cidrRanges.isEmpty() || !parsed.hostnames.isEmpty()) { + log.info("SSRF allowed hosts configured: {} CIDR range(s), {} hostname(s)", parsed.cidrRanges.size(), parsed.hostnames.size()); + } + } + + public static boolean isHostnameAllowed(String hostname) { + return allowedHosts.hostnames.contains(hostname.toLowerCase()); + } + + private static ParsedHostEntries parseHostEntries(List entries) { if (entries == null || entries.isEmpty()) { - additionalBlocked = AdditionalBlockedHosts.EMPTY; - return; + return ParsedHostEntries.EMPTY; } List cidrRanges = new ArrayList<>(); Set hostnames = new HashSet<>(); @@ -168,10 +207,9 @@ public class SsrfProtectionValidator { hostnames.add(trimmed.toLowerCase()); } } - additionalBlocked = new AdditionalBlockedHosts( + return new ParsedHostEntries( Collections.unmodifiableList(cidrRanges), Collections.unmodifiableSet(hostnames)); - log.info("SSRF additional blocked hosts configured: {} CIDR range(s), {} hostname(s)", cidrRanges.size(), hostnames.size()); } private static boolean isIpLiteral(String entry) { @@ -179,10 +217,18 @@ public class SsrfProtectionValidator { return !entry.isEmpty() && (Character.isDigit(entry.charAt(0)) || entry.contains(":")); } + private record ParsedHostEntries(List cidrRanges, Set hostnames) { + static final ParsedHostEntries EMPTY = new ParsedHostEntries(Collections.emptyList(), Collections.emptySet()); + } + record AdditionalBlockedHosts(List cidrRanges, Set hostnames) { static final AdditionalBlockedHosts EMPTY = new AdditionalBlockedHosts(Collections.emptyList(), Collections.emptySet()); } + record AllowedHosts(List cidrRanges, Set hostnames) { + static final AllowedHosts EMPTY = new AllowedHosts(Collections.emptyList(), Collections.emptySet()); + } + record CidrRange(byte[] network, int prefixLength) { static CidrRange of(String ip, int prefixLength) { diff --git a/common/util/src/test/java/org/thingsboard/common/util/SsrfProtectionValidatorTest.java b/common/util/src/test/java/org/thingsboard/common/util/SsrfProtectionValidatorTest.java index 6cb2d21a9a..77f915c09c 100644 --- a/common/util/src/test/java/org/thingsboard/common/util/SsrfProtectionValidatorTest.java +++ b/common/util/src/test/java/org/thingsboard/common/util/SsrfProtectionValidatorTest.java @@ -20,14 +20,16 @@ import org.junit.jupiter.api.parallel.ResourceLock; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.net.InetAddress; import java.net.URI; import java.util.Collections; import java.util.List; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatNoException; import static org.assertj.core.api.Assertions.assertThatThrownBy; -@ResourceLock("SsrfProtectionValidatorTest") // some tests mutate static additional-blocked-hosts +@ResourceLock("SsrfProtectionValidator") // to avoid race conditions when modifying SsrfProtectionValidator's static configuration public class SsrfProtectionValidatorTest { @ParameterizedTest @@ -335,4 +337,244 @@ public class SsrfProtectionValidatorTest { } } + // --- Allow-list tests --- + + @Test + void testAllowListCidrAllowsPrivateAddress() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of("192.168.1.0/24")); + // 192.168.1.1 is normally blocked (site-local), but allow-listed + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://192.168.1.1"), true)); + // Other private ranges remain blocked + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://10.0.0.1"), true)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testAllowListHostnameBypassesSuffixCheck() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of("my-device.local")); + // .local suffix is normally blocked, but allow-listed hostname passes + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://my-device.local/api"), true)); + // Other .local hostnames remain blocked + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://other-device.local/api"), true)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testAllowListPrecedenceOverBlockList() { + try { + // Block 8.8.8.0/24 via additional-blocked, but allow 8.8.8.8 via allow-list + SsrfProtectionValidator.setAdditionalBlockedHosts(List.of("8.8.8.0/24")); + SsrfProtectionValidator.setAllowedHosts(List.of("8.8.8.8")); + // Allow-list should win + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("https://8.8.8.8"), true)); + // Adjacent IP still blocked + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("https://8.8.8.9"), true)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + } finally { + SsrfProtectionValidator.setAdditionalBlockedHosts(Collections.emptyList()); + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testIsBlockedAddressPublicApi() throws Exception { + InetAddress loopback = InetAddress.getByName("127.0.0.1"); + assertThat(SsrfProtectionValidator.isBlockedAddress(loopback)).isTrue(); + + InetAddress publicIp = InetAddress.getByName("8.8.8.8"); + assertThat(SsrfProtectionValidator.isBlockedAddress(publicIp)).isFalse(); + + // Allow-listed private address + try { + SsrfProtectionValidator.setAllowedHosts(List.of("10.0.0.0/8")); + InetAddress privateIp = InetAddress.getByName("10.1.2.3"); + assertThat(SsrfProtectionValidator.isBlockedAddress(privateIp)).isFalse(); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testIsEnabledAccessor() { + boolean original = SsrfProtectionValidator.isEnabled(); + try { + SsrfProtectionValidator.setEnabled(true); + assertThat(SsrfProtectionValidator.isEnabled()).isTrue(); + SsrfProtectionValidator.setEnabled(false); + assertThat(SsrfProtectionValidator.isEnabled()).isFalse(); + } finally { + SsrfProtectionValidator.setEnabled(original); + } + } + + @Test + void testSetAllowedHostsEmptyAndNull() { + // Should not throw + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + SsrfProtectionValidator.setAllowedHosts(null); + } + + @Test + void testIsHostnameAllowed() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of("my-device.local", "Internal-Server.Corp")); + assertThat(SsrfProtectionValidator.isHostnameAllowed("my-device.local")).isTrue(); + assertThat(SsrfProtectionValidator.isHostnameAllowed("MY-DEVICE.LOCAL")).isTrue(); // case-insensitive + assertThat(SsrfProtectionValidator.isHostnameAllowed("internal-server.corp")).isTrue(); + assertThat(SsrfProtectionValidator.isHostnameAllowed("other-device.local")).isFalse(); + assertThat(SsrfProtectionValidator.isHostnameAllowed("example.com")).isFalse(); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testIsHostnameAllowedEmptyList() { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + assertThat(SsrfProtectionValidator.isHostnameAllowed("anything")).isFalse(); + } + + @Test + void testValidateUriUsesStaticEnabledFlag() { + boolean original = SsrfProtectionValidator.isEnabled(); + try { + // When enabled, loopback is blocked via the public one-arg overload + SsrfProtectionValidator.setEnabled(true); + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://127.0.0.1"))) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + + // When disabled, loopback passes + SsrfProtectionValidator.setEnabled(false); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://127.0.0.1"))); + } finally { + SsrfProtectionValidator.setEnabled(original); + } + } + + @Test + void testAllowListHostnameCaseInsensitive() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of("My-Device.LOCAL")); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://my-device.local/api"), true)); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://MY-DEVICE.LOCAL/api"), true)); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testAllowListOverridesCloudMetadataRange() { + try { + // 169.254.169.254 is link-local (blocked by default), allow-list should override + SsrfProtectionValidator.setAllowedHosts(List.of("169.254.169.254")); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://169.254.169.254/latest/meta-data/"), true)); + // Other link-local still blocked + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://169.254.1.1"), true)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testAllowListOverridesLoopback() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of("127.0.0.0/8")); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://127.0.0.1"), true)); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://127.1.2.3"), true)); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testAllowListCidrBoundary() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of("192.168.1.0/24")); + // Last address in range + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://192.168.1.255"), true)); + // First address outside range + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://192.168.2.0"), true)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + // Different subnet entirely + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://192.168.0.1"), true)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testBlockedIpv6UniqueLocal() throws Exception { + // fc00::/7 covers fc00:: through fdff:: + InetAddress fc00 = InetAddress.getByName("fc00::1"); + assertThat(SsrfProtectionValidator.isBlockedAddress(fc00)).isTrue(); + + InetAddress fdAddr = InetAddress.getByName("fd12:3456:789a::1"); + assertThat(SsrfProtectionValidator.isBlockedAddress(fdAddr)).isTrue(); + + // fe00:: is NOT in fc00::/7 (it's in fe80::/10 link-local, but fe00:: without the 80 bits is different) + // 2001:db8:: is a public documentation prefix, not blocked + InetAddress publicV6 = InetAddress.getByName("2001:db8::1"); + assertThat(SsrfProtectionValidator.isBlockedAddress(publicV6)).isFalse(); + } + + @Test + void testParseHostEntriesWithWhitespaceAndBlanks() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of(" 192.168.1.0/24 ", "", " ", "my-host.corp")); + // Trimmed CIDR works + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://192.168.1.1"), true)); + // Trimmed hostname works + assertThat(SsrfProtectionValidator.isHostnameAllowed("my-host.corp")).isTrue(); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testSetAllowedHostsReplacePrevious() { + try { + SsrfProtectionValidator.setAllowedHosts(List.of("192.168.1.0/24")); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://192.168.1.1"), true)); + + // Replace with different range + SsrfProtectionValidator.setAllowedHosts(List.of("10.0.0.0/8")); + // Old range no longer allowed + assertThatThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://192.168.1.1"), true)) + .isInstanceOf(RuntimeException.class) + .hasMessageContaining("URI is invalid"); + // New range allowed + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://10.1.2.3"), true)); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + + @Test + void testAllowListHostnameBypassesBlockedHostname() { + try { + // "localhost" is in BLOCKED_HOSTNAMES; allow-listing it should let it through + SsrfProtectionValidator.setAllowedHosts(List.of("localhost")); + assertThatNoException().isThrownBy(() -> SsrfProtectionValidator.validateUri(URI.create("http://localhost/path"), true)); + } finally { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + } + } diff --git a/common/version-control/pom.xml b/common/version-control/pom.xml index 432829d22c..3ea8054bd5 100644 --- a/common/version-control/pom.xml +++ b/common/version-control/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT common org.thingsboard.common diff --git a/dao/pom.xml b/dao/pom.xml index f153d3b848..6e67ea5d27 100644 --- a/dao/pom.xml +++ b/dao/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard dao @@ -280,7 +280,6 @@ org.apache.maven.plugins maven-jar-plugin - ${jar-plugin.version} diff --git a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java index a19ba6c0e7..4b469851cd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java +++ b/dao/src/main/java/org/thingsboard/server/dao/DaoUtil.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao; +import org.hibernate.exception.ConstraintViolationException; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; @@ -202,4 +203,13 @@ public final class DaoUtil { .collect(Collectors.toList()); } + public static ConstraintViolationException extractConstraintViolation(Throwable t) { + if (t instanceof ConstraintViolationException cve) { + return cve; + } else if (t != null && t.getCause() instanceof ConstraintViolationException cve) { + return cve; + } + return null; + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java index 96a86073d4..8e1408f4fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/NotificationRequestDao.java @@ -50,7 +50,7 @@ public interface NotificationRequestDao extends Dao { boolean existsByTenantIdAndStatusAndTemplateId(TenantId tenantId, NotificationRequestStatus status, NotificationTemplateId templateId); - int removeAllByCreatedTimeBefore(long ts); + int removeByTenantIdAndCreatedTimeBeforeBatch(TenantId tenantId, long ts, int batchSize); NotificationRequestInfo findInfoById(TenantId tenantId, NotificationRequestId id); diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java index 95515bd15e..54577ff9eb 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/BaseOtaPackageService.java @@ -196,7 +196,9 @@ public class BaseOtaPackageService extends AbstractCachedEntityService INCORRECT_OTA_PACKAGE_ID + id); try { + Long oid = getDataOidById(tenantId, otaPackageId); otaPackageDao.removeById(tenantId, otaPackageId.getId()); + unlinkDataIfPresent(tenantId, otaPackageId, oid); publishEvictEvent(new OtaPackageCacheEvictEvent(otaPackageId)); eventPublisher.publishEvent(DeleteEntityEvent.builder().tenantId(tenantId).entityId(otaPackageId).build()); } catch (Exception t) { @@ -215,6 +217,30 @@ public class BaseOtaPackageService extends AbstractCachedEntityService tenantOtaPackageRemover = - new PaginatedRemover<>() { - - @Override - protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { - return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); - } + private final PaginatedRemover tenantOtaPackageRemover = new PaginatedRemover<>() { + @Override + protected PageData findEntities(TenantId tenantId, TenantId id, PageLink pageLink) { + return otaPackageInfoDao.findOtaPackageInfoByTenantId(id, pageLink); + } - @Override - protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { - deleteOtaPackage(tenantId, entity.getId()); - } - }; + @Override + protected void removeEntity(TenantId tenantId, OtaPackageInfo entity) { + deleteOtaPackage(tenantId, entity.getId()); + } + }; @Override public Optional> findEntity(TenantId tenantId, EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java index c11a13cbe1..875aaea4ed 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/ota/OtaPackageDao.java @@ -18,15 +18,20 @@ package org.thingsboard.server.dao.ota; import org.thingsboard.server.common.data.OtaPackage; import org.thingsboard.server.common.data.id.OtaPackageId; import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.common.data.ota.OtaPackageType; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.ExportableEntityDao; import org.thingsboard.server.dao.TenantEntityWithDataDao; +import java.util.UUID; + public interface OtaPackageDao extends Dao, TenantEntityWithDataDao, ExportableEntityDao { Long sumDataSizeByTenantId(TenantId tenantId); OtaPackage findOtaPackageByTenantIdAndTitleAndVersion(TenantId tenantId, String title, String version); + Long getDataOidById(UUID id); + + Integer unlinkLargeObject(Long dataOid); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java index f88b672a34..37fe2950b4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rpc/RpcDao.java @@ -24,12 +24,13 @@ import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.Dao; public interface RpcDao extends Dao { + PageData findAllByDeviceId(TenantId tenantId, DeviceId deviceId, PageLink pageLink); PageData findAllByDeviceIdAndStatus(TenantId tenantId, DeviceId deviceId, RpcStatus rpcStatus, PageLink pageLink); PageData findAllRpcByTenantId(TenantId tenantId, PageLink pageLink); - int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime); + int deleteOutdatedRpcByTenantIdBatch(TenantId tenantId, Long expirationTime, int batchSize); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/Oauth2ClientDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/Oauth2ClientDataValidator.java index 07fbc06114..d5b965face 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/Oauth2ClientDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/Oauth2ClientDataValidator.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.service.validator; import lombok.AllArgsConstructor; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.SsrfProtectionValidator; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.oauth2.MapperType; @@ -28,6 +29,8 @@ import org.thingsboard.server.common.data.oauth2.TenantNameStrategyType; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.service.DataValidator; +import java.net.URI; + @Component @AllArgsConstructor public class Oauth2ClientDataValidator extends DataValidator { @@ -64,6 +67,11 @@ public class Oauth2ClientDataValidator extends DataValidator { if (StringUtils.isEmpty(customConfig.getUrl())) { throw new DataValidationException("Custom mapper URL should be specified!"); } + try { + SsrfProtectionValidator.validateUri(new URI(customConfig.getUrl())); + } catch (Exception e) { + throw new DataValidationException("Custom mapper URL is not allowed: " + e.getMessage()); + } } } } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java index 9d32e91ca2..f4cc406a75 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDao.java @@ -98,8 +98,8 @@ public class JpaNotificationRequestDao extends JpaAbstractDao findIdsByTenantId(@Param("tenantId") UUID tenantId, Pageable pageable); + // The 'data' column is of type OID (PostgreSQL large object reference), so it returns the OID as Long + @Query(value = "SELECT data FROM ota_package WHERE id = :id AND data IS NOT NULL", nativeQuery = true) + Long getDataOidById(@Param("id") UUID id); + + @Transactional + @Query(value = "SELECT lo_unlink(:oid)", nativeQuery = true) + Integer unlinkLargeObject(@Param("oid") Long oid); + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java index 9daae69d2a..a929b6d1e4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDao.java @@ -71,8 +71,8 @@ public class JpaRpcDao extends JpaAbstractDao implements RpcDao, @Transactional @Override - public int deleteOutdatedRpcByTenantId(TenantId tenantId, Long expirationTime) { - return rpcRepository.deleteOutdatedRpcByTenantId(tenantId.getId(), expirationTime); + public int deleteOutdatedRpcByTenantIdBatch(TenantId tenantId, Long expirationTime, int batchSize) { + return rpcRepository.deleteOutdatedRpcByTenantIdBatch(tenantId.getId(), expirationTime, batchSize); } @Override diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java index 8a9489333f..3f76170c84 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/rpc/RpcRepository.java @@ -21,20 +21,27 @@ import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import org.springframework.transaction.annotation.Transactional; import org.thingsboard.server.common.data.rpc.RpcStatus; import org.thingsboard.server.dao.model.sql.RpcEntity; import java.util.UUID; public interface RpcRepository extends JpaRepository { + Page findAllByTenantIdAndDeviceId(UUID tenantId, UUID deviceId, Pageable pageable); Page findAllByTenantIdAndDeviceIdAndStatus(UUID tenantId, UUID deviceId, RpcStatus status, Pageable pageable); Page findAllByTenantId(UUID tenantId, Pageable pageable); + @Transactional @Modifying - @Query(value = "DELETE FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime", + @Query(value = "DELETE FROM rpc WHERE id IN " + + "(SELECT id FROM rpc WHERE tenant_id = :tenantId AND created_time < :expirationTime LIMIT :batchSize)", nativeQuery = true) - int deleteOutdatedRpcByTenantId(@Param("tenantId") UUID tenantId, @Param("expirationTime") Long expirationTime); + int deleteOutdatedRpcByTenantIdBatch(@Param("tenantId") UUID tenantId, + @Param("expirationTime") Long expirationTime, + @Param("batchSize") int batchSize); + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java index a38499c82c..2f296d38f2 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/OtaPackageServiceTest.java @@ -37,13 +37,13 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.dao.device.DeviceProfileService; import org.thingsboard.server.dao.device.DeviceService; import org.thingsboard.server.dao.exception.DataValidationException; +import org.thingsboard.server.dao.ota.OtaPackageDao; import org.thingsboard.server.dao.ota.OtaPackageService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.dao.tenant.TenantProfileService; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -77,6 +77,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { @Autowired TenantProfileService tenantProfileService; @Autowired + OtaPackageDao otaPackageDao; + @Autowired TbTenantProfileCache tenantProfileCache; @Before @@ -118,10 +120,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { Assert.assertEquals(1, otaPackageService.sumDataSizeByTenantId(tenantId)); int maxSumDataSize = 8; - List packages = new ArrayList<>(maxSumDataSize); - for (int i = 2; i <= maxSumDataSize; i++) { - packages.add(createAndSaveFirmware(tenantId, "0." + i)); + createAndSaveFirmware(tenantId, "0." + i); Assert.assertEquals(i, otaPackageService.sumDataSizeByTenantId(tenantId)); } @@ -533,6 +533,39 @@ public class OtaPackageServiceTest extends AbstractServiceTest { Assert.assertNull(foundFirmware); } + @Test + public void testDeleteOtaPackageWithoutData() { + OtaPackageInfo firmwareInfo = new OtaPackageInfo(); + firmwareInfo.setTenantId(tenantId); + firmwareInfo.setDeviceProfileId(deviceProfileId); + firmwareInfo.setType(FIRMWARE); + firmwareInfo.setTitle(TITLE); + firmwareInfo.setVersion(VERSION); + OtaPackageInfo savedFirmwareInfo = otaPackageService.saveOtaPackageInfo(firmwareInfo, false); + + Assert.assertNotNull(savedFirmwareInfo); + Assert.assertNotNull(savedFirmwareInfo.getId()); + + // Should not throw NPE when deleting package without data (OID is null) + otaPackageService.deleteOtaPackage(tenantId, savedFirmwareInfo.getId()); + + OtaPackageInfo foundFirmware = otaPackageService.findOtaPackageInfoById(tenantId, savedFirmwareInfo.getId()); + Assert.assertNull(foundFirmware); + } + + @Test + public void testDeleteOtaPackageUnlinksLargeObject() { + OtaPackage savedFirmware = createAndSaveFirmware(tenantId, VERSION); + + Long oid = otaPackageDao.getDataOidById(savedFirmware.getId().getId()); + Assert.assertNotNull(oid); + + otaPackageService.deleteOtaPackage(tenantId, savedFirmware.getId()); + + // Verify the large object was unlinked - PostgreSQL throws an exception when the object doesn't exist + assertThatThrownBy(() -> otaPackageDao.unlinkLargeObject(oid)).hasMessageContaining("large object " + oid + " does not exist"); + } + @Test public void testFindTenantFirmwaresByTenantId() { List firmwares = new ArrayList<>(); @@ -567,8 +600,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { } } while (pageData.hasNext()); - Collections.sort(firmwares, idComparator); - Collections.sort(loadedFirmwares, idComparator); + firmwares.sort(idComparator); + loadedFirmwares.sort(idComparator); assertThat(firmwares).isEqualTo(loadedFirmwares); @@ -622,8 +655,8 @@ public class OtaPackageServiceTest extends AbstractServiceTest { } } while (pageData.hasNext()); - Collections.sort(firmwares, idComparator); - Collections.sort(loadedFirmwares, idComparator); + firmwares.sort(idComparator); + loadedFirmwares.sort(idComparator); assertThat(firmwares).isEqualTo(loadedFirmwares); @@ -726,4 +759,5 @@ public class OtaPackageServiceTest extends AbstractServiceTest { firmware.setDataSize(DATA_SIZE); return firmware; } + } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java index aa43826734..b66cdb4363 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/timeseries/nosql/TimeseriesServiceNoSqlTest.java @@ -65,7 +65,7 @@ public class TimeseriesServiceNoSqlTest extends BaseTimeseriesServiceTest { new BasicTsKvEntry(TimeUnit.MINUTES.toMillis(5), new JsonDataEntry("test", "{\"test\":\"testValue\"}"))); DeviceId deviceId = new DeviceId(Uuids.timeBased()); - tsService.save(tenantId, deviceId, timeseries, ttlInSec); + tsService.save(tenantId, deviceId, timeseries, ttlInSec).get(MAX_TIMEOUT, TimeUnit.SECONDS); List fullList = tsService.findAll(tenantId, deviceId, Collections.singletonList(new BaseReadTsKvQuery("test", 0L, TimeUnit.MINUTES.toMillis(6), 1000, 10, Aggregation.NONE))).get(MAX_TIMEOUT, TimeUnit.SECONDS); diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java new file mode 100644 index 0000000000..5e2fe2308b --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/notification/JpaNotificationRequestDaoTest.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.sql.notification; + +import org.junit.After; +import org.junit.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.id.NotificationRequestId; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.notification.NotificationRequest; +import org.thingsboard.server.common.data.notification.NotificationRequestStatus; +import org.thingsboard.server.dao.AbstractJpaDaoTest; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; + +public class JpaNotificationRequestDaoTest extends AbstractJpaDaoTest { + + @Autowired + JpaNotificationRequestDao notificationRequestDao; + + private final List createdRequests = new ArrayList<>(); + + @After + public void tearDown() { + for (NotificationRequest request : createdRequests) { + notificationRequestDao.removeById(request.getTenantId(), request.getId().getId()); + } + createdRequests.clear(); + } + + @Test + public void testBatchDeletion() { + TenantId sysTenantId = TenantId.SYS_TENANT_ID; + long now = System.currentTimeMillis(); + long oldTimestamp = now - TimeUnit.DAYS.toMillis(30); + + NotificationRequest oldRequest1 = createNotificationRequest(sysTenantId, oldTimestamp); + notificationRequestDao.save(sysTenantId, oldRequest1); + + NotificationRequest oldRequest2 = createNotificationRequest(sysTenantId, oldTimestamp); + notificationRequestDao.save(sysTenantId, oldRequest2); + + NotificationRequest freshRequest = createNotificationRequest(sysTenantId, now); + notificationRequestDao.save(sysTenantId, freshRequest); + + TenantId tenant2Id = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); + NotificationRequest tenant2Request = createNotificationRequest(tenant2Id, oldTimestamp); + notificationRequestDao.save(tenant2Id, tenant2Request); + + int batchSize = 10_000; + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, oldTimestamp - 1, batchSize)).isEqualTo(0); + + long expirationTime = now - TimeUnit.DAYS.toMillis(15); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(sysTenantId, expirationTime, batchSize)).isEqualTo(2); + + assertThat(notificationRequestDao.findById(sysTenantId, freshRequest.getId().getId())).isNotNull(); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2Id, now + 1, batchSize)).isEqualTo(1); + } + + @Test + public void testBatchDeletionWithSmallBatchSize() { + TenantId tenantId = TenantId.SYS_TENANT_ID; + long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); + + for (int i = 0; i < 10; i++) { + NotificationRequest request = createNotificationRequest(tenantId, oldTimestamp); + notificationRequestDao.save(tenantId, request); + } + + int batchSize = 3; + long expirationTime = System.currentTimeMillis(); + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(3); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(1); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenantId, expirationTime, batchSize)).isEqualTo(0); + } + + @Test + public void testBatchDeletionIsolationBetweenTenants() { + TenantId tenant1 = TenantId.SYS_TENANT_ID; + TenantId tenant2 = TenantId.fromUUID(UUID.fromString("3d193a7a-774b-4c05-84d5-f7fdcf7a37cf")); + long oldTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(30); + + for (int i = 0; i < 5; i++) { + NotificationRequest request = createNotificationRequest(tenant1, oldTimestamp); + notificationRequestDao.save(tenant1, request); + } + + for (int i = 0; i < 3; i++) { + NotificationRequest request = createNotificationRequest(tenant2, oldTimestamp); + notificationRequestDao.save(tenant2, request); + } + + int batchSize = 10_000; + long expirationTime = System.currentTimeMillis(); + + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant1, expirationTime, batchSize)).isEqualTo(5); + assertThat(notificationRequestDao.removeByTenantIdAndCreatedTimeBeforeBatch(tenant2, expirationTime, batchSize)).isEqualTo(3); + } + + private NotificationRequest createNotificationRequest(TenantId tenantId, long createdTime) { + NotificationRequest request = new NotificationRequest(); + request.setId(new NotificationRequestId(UUID.randomUUID())); + request.setTenantId(tenantId); + request.setCreatedTime(createdTime); + request.setTargets(List.of(UUID.randomUUID())); + request.setStatus(NotificationRequestStatus.SENT); + createdRequests.add(request); + return request; + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java index 1629922685..921339a92b 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/sql/rpc/JpaRpcDaoTest.java @@ -51,9 +51,10 @@ public class JpaRpcDaoTest extends AbstractJpaDaoTest { rpc.setDeviceId(new DeviceId(UUID.randomUUID())); rpcDao.saveAndFlush(rpc.getTenantId(), rpc); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, 0L)).isEqualTo(0); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(TenantId.SYS_TENANT_ID, Long.MAX_VALUE)).isEqualTo(2); - assertThat(rpcDao.deleteOutdatedRpcByTenantId(tenantId, System.currentTimeMillis() + 1)).isEqualTo(1); + int batchSize = 10_000; + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(TenantId.SYS_TENANT_ID, 0L, batchSize)).isEqualTo(0); + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(TenantId.SYS_TENANT_ID, Long.MAX_VALUE, batchSize)).isEqualTo(2); + assertThat(rpcDao.deleteOutdatedRpcByTenantIdBatch(tenantId, System.currentTimeMillis() + 1, batchSize)).isEqualTo(1); } } diff --git a/edqs/pom.xml b/edqs/pom.xml index 66782b8159..c1c471ef62 100644 --- a/edqs/pom.xml +++ b/edqs/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard edqs @@ -33,7 +33,6 @@ UTF-8 ${basedir}/.. java - false process-resources package tb-edqs @@ -179,10 +178,6 @@ org.apache.maven.plugins maven-assembly-plugin - - org.apache.maven.plugins - maven-install-plugin - org.xolstice.maven.plugins protobuf-maven-plugin diff --git a/monitoring/pom.xml b/monitoring/pom.xml index 7ad26adb61..e15b603449 100644 --- a/monitoring/pom.xml +++ b/monitoring/pom.xml @@ -21,7 +21,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard @@ -33,7 +33,6 @@ UTF-8 ${basedir}/.. java - false process-resources package tb-monitoring @@ -154,8 +153,8 @@ maven-assembly-plugin - org.apache.maven.plugins - maven-install-plugin + org.codehaus.mojo + build-helper-maven-plugin diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 5e37e274fa..8e4dcdc6c4 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -128,42 +128,50 @@ org.thingsboard.msa js-executor - docker-info + ${project.version} + pom org.thingsboard.msa web-ui - docker-info + ${project.version} + pom org.thingsboard.msa tb-node - docker-info + ${project.version} + pom org.thingsboard.msa.transport coap - docker-info + ${project.version} + pom org.thingsboard.msa.transport http - docker-info + ${project.version} + pom org.thingsboard.msa.transport mqtt - docker-info + ${project.version} + pom org.thingsboard.msa.transport lwm2m - docker-info + ${project.version} + pom org.thingsboard.msa.transport snmp - docker-info + ${project.version} + pom org.thingsboard.common @@ -206,7 +214,7 @@ org.apache.maven.surefire surefire-testng - ${surefire.version} + ${maven-surefire-plugin.version} diff --git a/msa/edqs/pom.xml b/msa/edqs/pom.xml index e483436bf1..3377cff6c6 100644 --- a/msa/edqs/pom.xml +++ b/msa/edqs/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../.. + ${maven.multiModuleProjectDirectory} tb-edqs tb-edqs /var/log/${pkg.name} @@ -46,8 +46,6 @@ org.thingsboard edqs ${project.version} - deb - deb provided @@ -56,25 +54,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-edqs - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard - edqs - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/edqs/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/edqs/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -99,36 +93,60 @@ + + copy-edqs + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/edqs/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -146,29 +164,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/js-executor/package.json b/msa/js-executor/package.json index 0efd464f30..a09596e6ac 100644 --- a/msa/js-executor/package.json +++ b/msa/js-executor/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-js-executor", "private": true, - "version": "4.2.2", + "version": "4.2.2.1", "description": "ThingsBoard JavaScript Executor Microservice", "main": "server.ts", "bin": "server.js", diff --git a/msa/js-executor/pom.xml b/msa/js-executor/pom.xml index c00257c0ac..87b3657d4a 100644 --- a/msa/js-executor/pom.xml +++ b/msa/js-executor/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -37,7 +37,6 @@ tb-js-executor tb-js-executor js - false process-resources package ${project.build.directory}/package/linux @@ -146,33 +145,37 @@ maven-assembly-plugin - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -222,29 +225,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/js-executor/yarn.lock b/msa/js-executor/yarn.lock index 6457be1de7..08a789e9b5 100644 --- a/msa/js-executor/yarn.lock +++ b/msa/js-executor/yarn.lock @@ -1036,9 +1036,9 @@ mimic-response@^3.1.0: integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== minimatch@^3.1.2: - version "3.1.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.3.tgz#6a5cba9b31f503887018f579c89f81f61162e624" - integrity sha512-M2GCs7Vk83NxkUyQV1bkABc4yxgz9kILhHImZiBPAZ9ybuvCb0/H7lEl5XvIg3g+9d4eNotkZA5IWwYl0tibaA== + version "3.1.5" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.5.tgz#580c88f8d5445f2bd6aa8f3cadefa0de79fbd69e" + integrity sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w== dependencies: brace-expansion "^1.1.7" @@ -1549,9 +1549,9 @@ tar-stream@^2.1.4: readable-stream "^3.1.1" tar@>=7.5.8, tar@^7.4.3: - version "7.5.9" - resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.9.tgz#817ac12a54bc4362c51340875b8985d7dc9724b8" - integrity sha512-BTLcK0xsDh2+PUe9F6c2TlRp4zOOBMTkoQHQIWSIzI0R7KG46uEwq4OPk2W7bZcprBMsuaeFsqwYr7pjh6CuHg== + version "7.5.11" + resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.11.tgz#1250fae45d98806b36d703b30973fa8e0a6d8868" + integrity sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ== dependencies: "@isaacs/fs-minipass" "^4.0.0" chownr "^3.0.0" diff --git a/msa/monitoring/pom.xml b/msa/monitoring/pom.xml index 62931e7e39..aded630179 100644 --- a/msa/monitoring/pom.xml +++ b/msa/monitoring/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa @@ -34,7 +34,7 @@ UTF-8 - ${basedir}/../.. + ${maven.multiModuleProjectDirectory} tb-monitoring tb-monitoring /var/log/${pkg.name} @@ -47,8 +47,6 @@ org.thingsboard monitoring ${project.version} - deb - deb provided @@ -57,25 +55,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-monitoring-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard - monitoring - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/monitoring/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/monitoring/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -100,36 +94,60 @@ + + copy-tb-monitoring-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/monitoring/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -147,29 +165,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/pom.xml b/msa/pom.xml index 83ec5318fa..d56a7aa265 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard msa @@ -31,11 +31,15 @@ ${basedir}/.. + + package thingsboard thingsboard/openjdk17:bookworm-slim true true - 1.4.13 + 3.5.1 none @@ -53,6 +57,37 @@ + + + skip-deb + + + pkg.skip.deb + true + + + + none + + + + + skip-pkg + + + pkg.skip + true + + + + none + + black-box-tests @@ -213,19 +248,12 @@ - - - com.spotify - dockerfile-maven-extension - ${dockerfile-maven.version} - - - com.spotify - dockerfile-maven-plugin - ${dockerfile-maven.version} + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} diff --git a/msa/tb-node/pom.xml b/msa/tb-node/pom.xml index ab7711890f..8272815640 100644 --- a/msa/tb-node/pom.xml +++ b/msa/tb-node/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../.. + ${maven.multiModuleProjectDirectory} thingsboard tb-node /var/log/${pkg.name} @@ -46,8 +46,6 @@ org.thingsboard application ${project.version} - deb - deb provided @@ -56,25 +54,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard - application - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/application/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/application/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -99,36 +93,60 @@ + + copy-tb-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/application/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -146,29 +164,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/tb/pom.xml b/msa/tb/pom.xml index b8adabf4bc..33e7abeabf 100644 --- a/msa/tb/pom.xml +++ b/msa/tb/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../.. + ${maven.multiModuleProjectDirectory} thingsboard tb-postgres tb-cassandra @@ -45,8 +45,6 @@ org.thingsboard application ${project.version} - deb - deb provided @@ -55,44 +53,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-postgres-deb - package - - copy - - - - - org.thingsboard - application - deb - deb - ${pkg.name}.deb - ${project.build.directory}/docker-postgres - - - - - - copy-tb-cassandra-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard - application - deb - deb - ${pkg.name}.deb - ${project.build.directory}/docker-cassandra - - + + + + ${main.dir}/application/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/application/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -101,7 +76,7 @@ org.apache.maven.plugins maven-resources-plugin - + copy-docker-tb-postgres-config process-resources @@ -141,64 +116,112 @@ + + copy-tb-postgres-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory}/docker-postgres + true + + + ${main.dir}/application/target + + ${pkg.name}.deb + + false + + + + + + copy-tb-cassandra-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory}/docker-cassandra + true + + + ${main.dir}/application/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-tb-postgres-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${tb-postgres.docker.name} - true - false - ${project.build.directory}/docker-postgres - true + docker + ${project.build.directory}/docker-postgres + + build + --no-cache + -t + ${docker.repo}/${tb-postgres.docker.name}:latest + . + tag-docker-tb-postgres-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${tb-postgres.docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${tb-postgres.docker.name}:latest + ${docker.repo}/${tb-postgres.docker.name}:${project.version} + build-docker-tb-cassandra-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${tb-cassandra.docker.name} - true - false - ${project.build.directory}/docker-cassandra - true + docker + ${project.build.directory}/docker-cassandra + + build + --no-cache + -t + ${docker.repo}/${tb-cassandra.docker.name}:latest + . + tag-docker-tb-cassandra-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${tb-cassandra.docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${tb-cassandra.docker.name}:latest + ${docker.repo}/${tb-cassandra.docker.name}:${project.version} + @@ -216,51 +239,55 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-tb-postgres-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${tb-postgres.docker.name} + docker + + push + ${docker.repo}/${tb-postgres.docker.name}:latest + push-version-docker-tb-postgres-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${tb-postgres.docker.name} + docker + + push + ${docker.repo}/${tb-postgres.docker.name}:${project.version} + push-latest-docker-tb-cassandra-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${tb-cassandra.docker.name} + docker + + push + ${docker.repo}/${tb-cassandra.docker.name}:latest + push-version-docker-tb-cassandra-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${tb-cassandra.docker.name} + docker + + push + ${docker.repo}/${tb-cassandra.docker.name}:${project.version} + diff --git a/msa/transport/coap/pom.xml b/msa/transport/coap/pom.xml index 7c86b9b8f3..6517d8b466 100644 --- a/msa/transport/coap/pom.xml +++ b/msa/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.msa.transport @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../../.. + ${maven.multiModuleProjectDirectory} tb-coap-transport tb-coap-transport /var/log/${pkg.name} @@ -46,8 +46,6 @@ org.thingsboard.transport coap ${project.version} - deb - deb provided @@ -56,25 +54,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-coap-transport-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard.transport - coap - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/transport/coap/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/transport/coap/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -99,36 +93,60 @@ + + copy-tb-coap-transport-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/transport/coap/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -146,29 +164,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/transport/http/pom.xml b/msa/transport/http/pom.xml index e214a36fe2..4c3445c4fc 100644 --- a/msa/transport/http/pom.xml +++ b/msa/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.msa.transport @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../../.. + ${maven.multiModuleProjectDirectory} tb-http-transport tb-http-transport /var/log/${pkg.name} @@ -46,8 +46,6 @@ org.thingsboard.transport http ${project.version} - deb - deb provided @@ -56,25 +54,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-http-transport-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard.transport - http - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/transport/http/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/transport/http/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -99,36 +93,60 @@ + + copy-tb-http-transport-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/transport/http/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -146,29 +164,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/transport/lwm2m/pom.xml b/msa/transport/lwm2m/pom.xml index a124e9605b..e106d81641 100644 --- a/msa/transport/lwm2m/pom.xml +++ b/msa/transport/lwm2m/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.msa.transport @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../../.. + ${maven.multiModuleProjectDirectory} tb-lwm2m-transport tb-lwm2m-transport /var/log/${pkg.name} @@ -46,8 +46,6 @@ org.thingsboard.transport lwm2m ${project.version} - deb - deb provided @@ -56,25 +54,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-lwm2m-transport-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard.transport - lwm2m - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/transport/lwm2m/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/transport/lwm2m/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -99,36 +93,60 @@ + + copy-tb-lwm2m-transport-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/transport/lwm2m/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -146,29 +164,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/transport/mqtt/pom.xml b/msa/transport/mqtt/pom.xml index 16ff52e8f2..f2f4f1e531 100644 --- a/msa/transport/mqtt/pom.xml +++ b/msa/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard.msa - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.msa.transport @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../../.. + ${maven.multiModuleProjectDirectory} tb-mqtt-transport tb-mqtt-transport /var/log/${pkg.name} @@ -46,8 +46,6 @@ org.thingsboard.transport mqtt ${project.version} - deb - deb provided @@ -56,25 +54,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-mqtt-transport-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard.transport - mqtt - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/transport/mqtt/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/transport/mqtt/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -99,36 +93,60 @@ + + copy-tb-mqtt-transport-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/transport/mqtt/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -146,29 +164,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/transport/pom.xml b/msa/transport/pom.xml index 2740e826f7..ae50472021 100644 --- a/msa/transport/pom.xml +++ b/msa/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa diff --git a/msa/transport/snmp/pom.xml b/msa/transport/snmp/pom.xml index 4d6dd82a6e..a1e024c735 100644 --- a/msa/transport/snmp/pom.xml +++ b/msa/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard.msa transport - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT org.thingsboard.msa.transport @@ -34,7 +34,7 @@ UTF-8 - ${basedir}/../../.. + ${maven.multiModuleProjectDirectory} tb-snmp-transport tb-snmp-transport /var/log/${pkg.name} @@ -47,8 +47,6 @@ org.thingsboard.transport snmp ${project.version} - deb - deb provided @@ -57,25 +55,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-snmp-transport-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard.transport - snmp - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/transport/snmp/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/transport/snmp/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -100,36 +94,60 @@ + + copy-tb-snmp-transport-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/transport/snmp/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -147,29 +165,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/vc-executor-docker/pom.xml b/msa/vc-executor-docker/pom.xml index 705dd77277..5713d9a142 100644 --- a/msa/vc-executor-docker/pom.xml +++ b/msa/vc-executor-docker/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -33,7 +33,7 @@ UTF-8 - ${basedir}/../.. + ${maven.multiModuleProjectDirectory} tb-vc-executor tb-vc-executor /var/log/${pkg.name} @@ -46,8 +46,6 @@ org.thingsboard.msa vc-executor ${project.version} - deb - deb provided @@ -56,25 +54,21 @@ org.apache.maven.plugins - maven-dependency-plugin + maven-enforcer-plugin - copy-tb-vc-executor-deb - package - - copy - + enforce-deb-exists + ${pkg.deb.phase} + enforce - - - org.thingsboard.msa - vc-executor - deb - deb - ${pkg.name}.deb - ${project.build.directory} - - + + + + ${main.dir}/msa/vc-executor/target/${pkg.name}.deb + + DEB artifact not found at ${main.dir}/msa/vc-executor/target/${pkg.name}.deb — build the upstream module first or pass -Dpkg.skip.deb=true. + + @@ -99,36 +93,60 @@ + + copy-tb-vc-executor-deb + ${pkg.deb.phase} + + copy-resources + + + ${project.build.directory} + true + + + ${main.dir}/msa/vc-executor/target + + ${pkg.name}.deb + + false + + + + - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -146,29 +164,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/vc-executor/pom.xml b/msa/vc-executor/pom.xml index b0397b1317..c476d1725c 100644 --- a/msa/vc-executor/pom.xml +++ b/msa/vc-executor/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -35,7 +35,6 @@ UTF-8 ${basedir}/../.. java - false process-resources package tb-vc-executor @@ -115,8 +114,8 @@ maven-assembly-plugin - org.apache.maven.plugins - maven-install-plugin + org.codehaus.mojo + build-helper-maven-plugin diff --git a/msa/web-ui/config/custom-environment-variables.yml b/msa/web-ui/config/custom-environment-variables.yml index 90bce53cb4..6ba9147555 100644 --- a/msa/web-ui/config/custom-environment-variables.yml +++ b/msa/web-ui/config/custom-environment-variables.yml @@ -25,6 +25,20 @@ thingsboard: host: "TB_HOST" # ThingsBoard node port port: "TB_PORT" +security: + headers: + x-content-type-options: + enabled: "SECURITY_HEADERS_X_CONTENT_TYPE_OPTIONS_ENABLED" + referrer-policy: + enabled: "SECURITY_HEADERS_REFERRER_POLICY_ENABLED" + value: "SECURITY_HEADERS_REFERRER_POLICY_VALUE" + x-frame-options: + enabled: "SECURITY_HEADERS_X_FRAME_OPTIONS_ENABLED" + value: "SECURITY_HEADERS_X_FRAME_OPTIONS_VALUE" + content-security-policy: + enabled: "SECURITY_HEADERS_CONTENT_SECURITY_POLICY_ENABLED" + value: "SECURITY_HEADERS_CONTENT_SECURITY_POLICY_VALUE" + report-only: "SECURITY_HEADERS_CONTENT_SECURITY_POLICY_REPORT_ONLY" logger: level: "LOGGER_LEVEL" path: "LOG_FOLDER" diff --git a/msa/web-ui/config/default.yml b/msa/web-ui/config/default.yml index b26424a8da..6ffa85b3bc 100644 --- a/msa/web-ui/config/default.yml +++ b/msa/web-ui/config/default.yml @@ -25,6 +25,20 @@ thingsboard: host: "localhost" # ThingsBoard node port port: "8080" +security: + headers: + x-content-type-options: + enabled: true + referrer-policy: + enabled: true + value: "strict-origin-when-cross-origin" + x-frame-options: + enabled: false + value: "SAMEORIGIN" + content-security-policy: + enabled: false + value: "" + report-only: false logger: level: "info" path: "logs" diff --git a/msa/web-ui/package.json b/msa/web-ui/package.json index 87f09128f6..1a956617cd 100644 --- a/msa/web-ui/package.json +++ b/msa/web-ui/package.json @@ -1,7 +1,7 @@ { "name": "thingsboard-web-ui", "private": true, - "version": "4.2.2", + "version": "4.2.2.1", "description": "ThingsBoard Web UI Microservice", "main": "server.ts", "bin": "server.js", diff --git a/msa/web-ui/pom.xml b/msa/web-ui/pom.xml index 59d22b47c8..2307818944 100644 --- a/msa/web-ui/pom.xml +++ b/msa/web-ui/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT msa org.thingsboard.msa @@ -39,7 +39,6 @@ tb-web-ui tb-web-ui js - false process-resources package ${project.build.directory}/package/linux @@ -193,33 +192,37 @@ maven-assembly-plugin - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin build-docker-image pre-integration-test - - build - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - true - false - ${project.build.directory} + docker + ${project.build.directory} + + build + -t + ${docker.repo}/${docker.name}:latest + . + tag-docker-image pre-integration-test - - tag - + exec ${dockerfile.skip} - ${docker.repo}/${docker.name} - ${project.version} + docker + + tag + ${docker.repo}/${docker.name}:latest + ${docker.repo}/${docker.name}:${project.version} + @@ -269,29 +272,31 @@ - com.spotify - dockerfile-maven-plugin + org.codehaus.mojo + exec-maven-plugin push-latest-docker-image pre-integration-test - - push - + exec - latest - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:latest + push-version-docker-image pre-integration-test - - push - + exec - ${project.version} - ${docker.repo}/${docker.name} + docker + + push + ${docker.repo}/${docker.name}:${project.version} + diff --git a/msa/web-ui/server.ts b/msa/web-ui/server.ts index 68baf5079f..0a4ddbfa6b 100644 --- a/msa/web-ui/server.ts +++ b/msa/web-ui/server.ts @@ -60,6 +60,36 @@ let connections: Socket[] = []; const app = express(); server = http.createServer(app); + // Build security headers map once at startup. + // node-config passes env var overrides as strings, so enabled can be boolean or string. + const isEnabled = (val: any) => val === true || val === 'true'; + const securityHeaders: Record = {}; + const hc: any = config.get('security.headers'); + if (isEnabled(hc['x-content-type-options']?.enabled)) { + securityHeaders['X-Content-Type-Options'] = 'nosniff'; + } + if (isEnabled(hc['referrer-policy']?.enabled)) { + securityHeaders['Referrer-Policy'] = hc['referrer-policy']?.value || 'strict-origin-when-cross-origin'; + } + if (isEnabled(hc['x-frame-options']?.enabled)) { + securityHeaders['X-Frame-Options'] = hc['x-frame-options']?.value || 'SAMEORIGIN'; + } + if (isEnabled(hc['content-security-policy']?.enabled) && hc['content-security-policy']?.value) { + const csp = hc['content-security-policy']; + const name = isEnabled(csp['report-only']) + ? 'Content-Security-Policy-Report-Only' : 'Content-Security-Policy'; + securityHeaders[name] = csp.value; + } + logger.info('Security headers: %s', JSON.stringify(securityHeaders)); + + // Apply security headers to all responses + app.use((_req, res, next) => { + for (const [name, value] of Object.entries(securityHeaders)) { + res.setHeader(name, value); + } + next(); + }); + let apiProxy: httpProxy; if (useApiProxy) { apiProxy = httpProxy.createProxyServer({ diff --git a/msa/web-ui/yarn.lock b/msa/web-ui/yarn.lock index 11a708d8bf..52dd2a03f3 100644 --- a/msa/web-ui/yarn.lock +++ b/msa/web-ui/yarn.lock @@ -1098,9 +1098,9 @@ mimic-response@^3.1.0: integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== minimatch@^3.1.2: - version "3.1.3" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.3.tgz#6a5cba9b31f503887018f579c89f81f61162e624" - integrity sha512-M2GCs7Vk83NxkUyQV1bkABc4yxgz9kILhHImZiBPAZ9ybuvCb0/H7lEl5XvIg3g+9d4eNotkZA5IWwYl0tibaA== + version "3.1.5" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.5.tgz#580c88f8d5445f2bd6aa8f3cadefa0de79fbd69e" + integrity sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w== dependencies: brace-expansion "^1.1.7" @@ -1631,9 +1631,9 @@ tar-stream@^2.1.4: readable-stream "^3.1.1" tar@>=7.5.8, tar@^7.4.3: - version "7.5.9" - resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.9.tgz#817ac12a54bc4362c51340875b8985d7dc9724b8" - integrity sha512-BTLcK0xsDh2+PUe9F6c2TlRp4zOOBMTkoQHQIWSIzI0R7KG46uEwq4OPk2W7bZcprBMsuaeFsqwYr7pjh6CuHg== + version "7.5.11" + resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.11.tgz#1250fae45d98806b36d703b30973fa8e0a6d8868" + integrity sha512-ChjMH33/KetonMTAtpYdgUFr0tbz69Fp2v7zWxQfYZX4g5ZN2nOBXm1R2xyA+lMIKrLKIoKAwFj93jE/avX9cQ== dependencies: "@isaacs/fs-minipass" "^4.0.0" chownr "^3.0.0" diff --git a/netty-mqtt/pom.xml b/netty-mqtt/pom.xml index fd5598f83f..615a0ed728 100644 --- a/netty-mqtt/pom.xml +++ b/netty-mqtt/pom.xml @@ -19,11 +19,11 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard netty-mqtt - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT jar Netty MQTT Client @@ -125,7 +125,6 @@ org.apache.maven.plugins maven-jar-plugin - 3.1.1 diff --git a/packaging/java/build.gradle b/packaging/java/build.gradle index 5ef4576a91..16d3db5131 100644 --- a/packaging/java/build.gradle +++ b/packaging/java/build.gradle @@ -173,3 +173,6 @@ buildDeb { link("${pkgInstallFolder}/bin/${pkgName}.yml", "${pkgInstallFolder}/conf/${pkgName}.yml") link("/etc/${pkgName}/conf", "${pkgInstallFolder}/conf") } + +buildDeb.onlyIf { findProperty('skipDeb') != 'true' } +buildRpm.onlyIf { findProperty('skipRpm') != 'true' } diff --git a/packaging/js/build.gradle b/packaging/js/build.gradle index 15b8e546a1..009d1aa460 100644 --- a/packaging/js/build.gradle +++ b/packaging/js/build.gradle @@ -126,3 +126,6 @@ buildDeb { link("/etc/${pkgName}/conf", "${pkgInstallFolder}/conf") } + +buildDeb.onlyIf { findProperty('skipDeb') != 'true' } +buildRpm.onlyIf { findProperty('skipRpm') != 'true' } diff --git a/pom.xml b/pom.xml index f00718c322..9dd86011be 100755 --- a/pom.xml +++ b/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT pom Thingsboard @@ -31,18 +31,41 @@ 17 17 ${basedir} - true none none + + + false + + false + + false + + false + + ${pkg.package.phase} thingsboard ${project.name} /var/log/${pkg.name} /usr/share/${pkg.name} - 3.4.13 - 10.1.52 - 2.18.6 + 3.5.12 + 2.21.1 + 4.1.132.Final 2.4.0-b180830.0359 - 5.1.5 0.12.5 0.10 4.17.0 @@ -63,16 +86,18 @@ 2.0.0-M15 2.0.1 5.6.0 - 3.9.3 + 3.9.5 3.25.5 1.76.0 1.2.9 - 1.18.38 + 1.18.44 1.2.5 1.2.5 1.7.1 - 3.5.4 - 3.4.0 + 3.5.0 + 3.5.4 + 3.1.4 + 3.1.4 2.8.8TB 2.2.30 0.8 @@ -91,7 +116,6 @@ 1.10.1 8.10.1 3.5.3 - 2.2 1.12.701 1.128.1 @@ -183,6 +207,43 @@ + + + skip-deb + + + pkg.skip.deb + true + + + + none + + + + + + skip-pkg + + + pkg.skip + true + + + + true + true + true + true + none + + + packaging @@ -488,7 +549,12 @@ org.springframework.boot spring-boot-maven-plugin - ${pkg.disabled} + + ${pkg.skip.bootjar} ${pkg.mainClass} boot ZIP @@ -532,6 +598,10 @@ -PpkgInstallFolder=${pkg.installFolder} -PpkgCopyInstallScripts=${pkg.copyInstallScripts} -PpkgLogFolder=${pkg.unixLogFolder} + -PskipDeb=${pkg.skip.deb} + -PskipRpm=${pkg.skip.rpm} + --project-cache-dir + ${project.build.directory}/.gradle --warning-mode all @@ -549,6 +619,7 @@ org.apache.maven.plugins maven-assembly-plugin + ${pkg.skip.zip} ${pkg.name} ${main.dir}/packaging/${pkg.type}/assembly/windows.xml @@ -565,23 +636,24 @@ - org.apache.maven.plugins - maven-install-plugin - - ${project.build.directory}/${pkg.name}.deb - ${project.artifactId} - ${project.groupId} - ${project.version} - deb - deb - + org.codehaus.mojo + build-helper-maven-plugin - install-deb - ${pkg.package.phase} + attach-deb + ${pkg.deb.phase} - install-file + attach-artifact + + + + ${project.build.directory}/${pkg.name}.deb + deb + deb + + + @@ -635,7 +707,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.1.1 + ${jar-plugin.version} org.apache.maven.plugins @@ -655,12 +727,12 @@ com.github.eirslett frontend-maven-plugin - 1.12.0 + 2.0.0 org.apache.maven.plugins maven-surefire-plugin - ${surefire.version} + ${maven-surefire-plugin.version} -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=200 @@ -673,12 +745,17 @@ org.apache.maven.plugins maven-install-plugin - 3.0.0-M1 + ${maven-install-plugin.version} org.apache.maven.plugins maven-deploy-plugin - 3.0.0-M1 + ${maven-deploy-plugin.version} + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.5.0 org.codehaus.mojo @@ -888,6 +965,21 @@ com.mycila license-maven-plugin + + org.apache.maven.plugins + maven-clean-plugin + false + + + + ${main.dir}/packaging/java/.gradle + + + ${main.dir}/packaging/js/.gradle + + + + @@ -901,34 +993,24 @@ - - - org.apache.tomcat.embed - tomcat-embed-core - ${tomcat.version} - - - org.apache.tomcat.embed - tomcat-embed-el - ${tomcat.version} - - - org.apache.tomcat.embed - tomcat-embed-websocket - ${tomcat.version} - - - - + com.fasterxml.jackson jackson-bom - ${jackson.version} + ${jackson-bom.version} pom import - - + + + + io.netty + netty-bom + ${netty.version} + pom + import + + org.springframework.boot spring-boot-dependencies @@ -1202,11 +1284,6 @@ jjwt ${jjwt.version} - - org.yaml - snakeyaml - ${snakeyaml.version} - antlr antlr @@ -1507,12 +1584,6 @@ ${dbunit.version} test - - org.projectlombok - lombok - ${lombok.version} - provided - org.eclipse.paho org.eclipse.paho.client.mqttv3 @@ -1558,11 +1629,6 @@ bcprov-ext-jdk18on ${bouncycastle.version} - - redis.clients - jedis - ${jedis.version} - com.sun.winsw winsw diff --git a/rest-client/pom.xml b/rest-client/pom.xml index 11c1ba6d8b..787a28a497 100644 --- a/rest-client/pom.xml +++ b/rest-client/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard rest-client diff --git a/rule-engine/pom.xml b/rule-engine/pom.xml index 1660eb9f4d..ff443deb61 100644 --- a/rule-engine/pom.xml +++ b/rule-engine/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard rule-engine diff --git a/rule-engine/rule-engine-api/pom.xml b/rule-engine/rule-engine-api/pom.xml index 669491baef..bcc68f2049 100644 --- a/rule-engine/rule-engine-api/pom.xml +++ b/rule-engine/rule-engine-api/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/rule-engine/rule-engine-components/pom.xml b/rule-engine/rule-engine-components/pom.xml index c396b560c6..c326b93b7c 100644 --- a/rule-engine/rule-engine-components/pom.xml +++ b/rule-engine/rule-engine-components/pom.xml @@ -22,7 +22,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT rule-engine org.thingsboard.rule-engine diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/SsrfSafeAddressResolverGroup.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/SsrfSafeAddressResolverGroup.java new file mode 100644 index 0000000000..9d15cb9793 --- /dev/null +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/SsrfSafeAddressResolverGroup.java @@ -0,0 +1,174 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.rest; + +import io.netty.resolver.AddressResolver; +import io.netty.resolver.AddressResolverGroup; +import io.netty.resolver.DefaultAddressResolverGroup; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Future; +import io.netty.util.concurrent.Promise; +import org.thingsboard.common.util.SsrfProtectionValidator; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Custom Netty {@link AddressResolverGroup} that validates every resolved IP address + * against the SSRF block-list at connection time. This eliminates the DNS rebinding + * TOCTOU gap where a hostname resolves to a safe IP during validation but to a + * private/metadata IP when the actual connection is made. + *

+ * Only wired into {@link TbHttpClient} when SSRF protection is enabled. + */ +public final class SsrfSafeAddressResolverGroup extends AddressResolverGroup { + + public static final SsrfSafeAddressResolverGroup INSTANCE = new SsrfSafeAddressResolverGroup(); + + private SsrfSafeAddressResolverGroup() { + } + + @Override + protected AddressResolver newResolver(EventExecutor executor) throws Exception { + AddressResolver delegate = DefaultAddressResolverGroup.INSTANCE.getResolver(executor); + return new SsrfValidatingResolver(executor, delegate); + } + + private static final class SsrfValidatingResolver implements AddressResolver { + + private final EventExecutor executor; + private final AddressResolver delegate; + + SsrfValidatingResolver(EventExecutor executor, AddressResolver delegate) { + this.executor = executor; + this.delegate = delegate; + } + + @Override + public boolean isSupported(SocketAddress address) { + return delegate.isSupported(address); + } + + @Override + public boolean isResolved(SocketAddress address) { + return delegate.isResolved(address); + } + + @Override + public Future resolve(SocketAddress address) { + return resolve(address, executor.newPromise()); + } + + @Override + public Future resolve(SocketAddress address, Promise promise) { + delegate.resolve(address).addListener((Future future) -> { + try { + if (!future.isSuccess()) { + promise.tryFailure(future.cause()); + return; + } + InetSocketAddress resolved = future.getNow(); + if (isOriginalHostAllowed(address)) { + promise.trySuccess(resolved); + } else if (isBlocked(resolved)) { + promise.tryFailure(new RuntimeException( + "URI is invalid: host '" + getHostString(address) + "' is not allowed")); + } else { + promise.trySuccess(resolved); + } + } catch (Exception e) { + promise.tryFailure(e); + } + }); + return promise; + } + + @Override + public Future> resolveAll(SocketAddress address) { + return resolveAll(address, executor.newPromise()); + } + + @Override + public Future> resolveAll(SocketAddress address, Promise> promise) { + delegate.resolveAll(address).addListener((Future> future) -> { + try { + if (!future.isSuccess()) { + promise.tryFailure(future.cause()); + return; + } + List resolved = future.getNow(); + if (isOriginalHostAllowed(address)) { + promise.trySuccess(resolved); + return; + } + Set blocked = null; + for (InetSocketAddress addr : resolved) { + if (isBlocked(addr)) { + if (blocked == null) { + blocked = new HashSet<>(2); + } + blocked.add(addr); + } + } + if (blocked == null) { + promise.trySuccess(resolved); + } else if (blocked.size() == resolved.size()) { + promise.tryFailure(new RuntimeException( + "URI is invalid: host '" + getHostString(address) + "' is not allowed")); + } else { + List safe = new ArrayList<>(resolved.size() - blocked.size()); + for (InetSocketAddress addr : resolved) { + if (!blocked.contains(addr)) { + safe.add(addr); + } + } + promise.trySuccess(safe); + } + } catch (Exception e) { + promise.tryFailure(e); + } + }); + return promise; + } + + @Override + public void close() { + delegate.close(); + } + + private static boolean isBlocked(InetSocketAddress socketAddress) { + InetAddress addr = socketAddress.getAddress(); + return addr != null && SsrfProtectionValidator.isBlockedAddress(addr); + } + + private static boolean isOriginalHostAllowed(SocketAddress address) { + if (address instanceof InetSocketAddress isa) { + String host = isa.getHostString(); + return host != null && SsrfProtectionValidator.isHostnameAllowed(host); + } + return false; + } + + private static String getHostString(SocketAddress address) { + return address instanceof InetSocketAddress isa ? isa.getHostString() : address.toString(); + } + } +} diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 24f88e88a3..df9ce0194b 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -103,6 +103,7 @@ public class TbHttpClient { .build(); HttpClient httpClient = HttpClient.create(connectionProvider) + .followRedirect(false) .runOn(getSharedOrCreateEventLoopGroup(eventLoopGroupShared)) .doOnConnected(c -> c.addHandlerLast(new ReadTimeoutHandler(config.getReadTimeoutMs(), TimeUnit.MILLISECONDS))); @@ -138,6 +139,10 @@ public class TbHttpClient { httpClient = httpClient.secure(t -> t.sslContext(sslContext)); } + if (SsrfProtectionValidator.isEnabled()) { + httpClient = httpClient.resolver(SsrfSafeAddressResolverGroup.INSTANCE); + } + validateMaxInMemoryBufferSize(config); this.webClient = WebClient.builder() diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/SsrfSafeAddressResolverGroupTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/SsrfSafeAddressResolverGroupTest.java new file mode 100644 index 0000000000..fc3b825d36 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/SsrfSafeAddressResolverGroupTest.java @@ -0,0 +1,161 @@ +/** + * Copyright © 2016-2026 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.rule.engine.rest; + +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.resolver.AddressResolver; +import io.netty.util.concurrent.EventExecutor; +import io.netty.util.concurrent.Promise; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceLock; +import org.thingsboard.common.util.SsrfProtectionValidator; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@ResourceLock("SsrfProtectionValidator") // to avoid race conditions when modifying SsrfProtectionValidator's static configuration +class SsrfSafeAddressResolverGroupTest { + + private static NioEventLoopGroup eventLoopGroup; + + @BeforeAll + static void setUp() { + eventLoopGroup = new NioEventLoopGroup(1); + } + + @AfterAll + static void tearDown() { + eventLoopGroup.shutdownGracefully(0, 5, TimeUnit.SECONDS); + SsrfProtectionValidator.setEnabled(false); + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + + @BeforeEach + void enableSsrf() { + SsrfProtectionValidator.setEnabled(true); + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + } + + @AfterEach + void resetState() { + SsrfProtectionValidator.setAllowedHosts(Collections.emptyList()); + SsrfProtectionValidator.setEnabled(false); + } + + @Test + void isBlockedAddressWorksForLoopback() throws Exception { + assertThat(SsrfProtectionValidator.isBlockedAddress(InetAddress.getByName("127.0.0.1"))).isTrue(); + assertThat(SsrfProtectionValidator.isBlockedAddress(InetAddress.getByName("192.168.1.1"))).isTrue(); + assertThat(SsrfProtectionValidator.isBlockedAddress(InetAddress.getByName("8.8.8.8"))).isFalse(); + } + + @Test + void resolvePublicIpSucceeds() throws Exception { + EventExecutor executor = eventLoopGroup.next(); + AddressResolver resolver = SsrfSafeAddressResolverGroup.INSTANCE.getResolver(executor); + Promise promise = executor.newPromise(); + + executor.submit(() -> resolver.resolve(InetSocketAddress.createUnresolved("8.8.8.8", 80), promise)); + InetSocketAddress result = promise.get(10, TimeUnit.SECONDS); + + assertThat(result.getAddress()).isNotNull(); + assertThat(result.getAddress().getHostAddress()).isEqualTo("8.8.8.8"); + } + + @Test + void resolveLoopbackFailsWhenSsrfEnabled() throws Exception { + assertThat(SsrfProtectionValidator.isEnabled()).isTrue(); + + EventExecutor executor = eventLoopGroup.next(); + AddressResolver resolver = SsrfSafeAddressResolverGroup.INSTANCE.getResolver(executor); + Promise promise = executor.newPromise(); + + executor.submit(() -> resolver.resolve(InetSocketAddress.createUnresolved("127.0.0.1", 80), promise)); + + assertThatThrownBy(() -> promise.get(10, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class) + .hasRootCauseInstanceOf(RuntimeException.class) + .rootCause().hasMessageContaining("is not allowed"); + } + + @Test + void resolvePrivateIpFailsWhenSsrfEnabled() throws Exception { + assertThat(SsrfProtectionValidator.isEnabled()).isTrue(); + + EventExecutor executor = eventLoopGroup.next(); + AddressResolver resolver = SsrfSafeAddressResolverGroup.INSTANCE.getResolver(executor); + Promise promise = executor.newPromise(); + + executor.submit(() -> resolver.resolve(InetSocketAddress.createUnresolved("192.168.1.1", 80), promise)); + + assertThatThrownBy(() -> promise.get(10, TimeUnit.SECONDS)) + .isInstanceOf(ExecutionException.class) + .hasRootCauseInstanceOf(RuntimeException.class) + .rootCause().hasMessageContaining("is not allowed"); + } + + @Test + void resolveAllowedPrivateIpSucceeds() throws Exception { + SsrfProtectionValidator.setAllowedHosts(List.of("192.168.1.0/24")); + + EventExecutor executor = eventLoopGroup.next(); + AddressResolver resolver = SsrfSafeAddressResolverGroup.INSTANCE.getResolver(executor); + Promise promise = executor.newPromise(); + + executor.submit(() -> resolver.resolve(InetSocketAddress.createUnresolved("192.168.1.1", 80), promise)); + InetSocketAddress result = promise.get(10, TimeUnit.SECONDS); + + assertThat(result.getAddress().getHostAddress()).isEqualTo("192.168.1.1"); + } + + @Test + void resolveAllPublicIpSucceeds() throws Exception { + EventExecutor executor = eventLoopGroup.next(); + AddressResolver resolver = SsrfSafeAddressResolverGroup.INSTANCE.getResolver(executor); + Promise> promise = executor.newPromise(); + + executor.submit(() -> resolver.resolveAll(InetSocketAddress.createUnresolved("8.8.8.8", 80), promise)); + List results = promise.get(10, TimeUnit.SECONDS); + + assertThat(results).isNotEmpty(); + assertThat(results.get(0).getAddress().getHostAddress()).isEqualTo("8.8.8.8"); + } + + @Test + void resolveAllPrivateIpFailsWhenSsrfEnabled() { + assertThatThrownBy(() -> { + EventExecutor executor = eventLoopGroup.next(); + AddressResolver resolver = SsrfSafeAddressResolverGroup.INSTANCE.getResolver(executor); + Promise> promise = executor.newPromise(); + executor.submit(() -> resolver.resolveAll(InetSocketAddress.createUnresolved("127.0.0.1", 80), promise)); + promise.get(10, TimeUnit.SECONDS); + }).isInstanceOf(ExecutionException.class) + .hasRootCauseInstanceOf(RuntimeException.class) + .rootCause().hasMessageContaining("is not allowed"); + } + +} diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java index 6363e1032a..a33d25b038 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.ResourceLock; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockserver.integration.ClientAndServer; @@ -54,6 +55,7 @@ import static org.mockserver.integration.ClientAndServer.startClientAndServer; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; +@ResourceLock("SsrfProtectionValidator") // to avoid race conditions when modifying SsrfProtectionValidator's static configuration public class TbHttpClientTest { EventLoopGroup eventLoop; diff --git a/tools/pom.xml b/tools/pom.xml index 5a4db71d84..339be85998 100644 --- a/tools/pom.xml +++ b/tools/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard tools diff --git a/transport/coap/pom.xml b/transport/coap/pom.xml index ed7b04f312..6e67ff417e 100644 --- a/transport/coap/pom.xml +++ b/transport/coap/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.transport @@ -34,7 +34,6 @@ UTF-8 ${basedir}/../.. java - false process-resources package tb-coap-transport @@ -109,8 +108,8 @@ maven-assembly-plugin - org.apache.maven.plugins - maven-install-plugin + org.codehaus.mojo + build-helper-maven-plugin diff --git a/transport/http/pom.xml b/transport/http/pom.xml index 5becbfec94..39b57240fb 100644 --- a/transport/http/pom.xml +++ b/transport/http/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.transport @@ -34,7 +34,6 @@ UTF-8 ${basedir}/../.. java - false process-resources package tb-http-transport @@ -109,8 +108,8 @@ maven-assembly-plugin - org.apache.maven.plugins - maven-install-plugin + org.codehaus.mojo + build-helper-maven-plugin diff --git a/transport/lwm2m/pom.xml b/transport/lwm2m/pom.xml index e116949ce9..ebe1e6bcca 100644 --- a/transport/lwm2m/pom.xml +++ b/transport/lwm2m/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.transport @@ -34,7 +34,6 @@ UTF-8 ${basedir}/../.. java - false process-resources package tb-lwm2m-transport @@ -163,8 +162,8 @@ maven-assembly-plugin - org.apache.maven.plugins - maven-install-plugin + org.codehaus.mojo + build-helper-maven-plugin diff --git a/transport/mqtt/pom.xml b/transport/mqtt/pom.xml index d73330908a..856df50aa0 100644 --- a/transport/mqtt/pom.xml +++ b/transport/mqtt/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport org.thingsboard.transport @@ -34,7 +34,6 @@ UTF-8 ${basedir}/../.. java - false process-resources package tb-mqtt-transport @@ -109,8 +108,8 @@ maven-assembly-plugin - org.apache.maven.plugins - maven-install-plugin + org.codehaus.mojo + build-helper-maven-plugin diff --git a/transport/pom.xml b/transport/pom.xml index 49d35b493b..96ddd89a8f 100644 --- a/transport/pom.xml +++ b/transport/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard transport diff --git a/transport/snmp/pom.xml b/transport/snmp/pom.xml index 94502dbf3b..f57e99b113 100644 --- a/transport/snmp/pom.xml +++ b/transport/snmp/pom.xml @@ -21,7 +21,7 @@ org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT transport @@ -36,7 +36,6 @@ UTF-8 ${basedir}/../.. java - false process-resources package tb-snmp-transport @@ -94,8 +93,8 @@ maven-assembly-plugin - org.apache.maven.plugins - maven-install-plugin + org.codehaus.mojo + build-helper-maven-plugin diff --git a/ui-ngx/package.json b/ui-ngx/package.json index b46ba7912b..d479a75a5d 100644 --- a/ui-ngx/package.json +++ b/ui-ngx/package.json @@ -1,6 +1,6 @@ { "name": "thingsboard", - "version": "4.2.2", + "version": "4.2.2.1", "scripts": { "ng": "ng", "start": "node --max_old_space_size=8048 ./node_modules/@angular/cli/bin/ng serve --configuration development --host 0.0.0.0 --open", @@ -13,16 +13,16 @@ }, "private": true, "dependencies": { - "@angular/animations": "20.3.17", + "@angular/animations": "20.3.18", "@angular/cdk": "20.2.14", - "@angular/common": "20.3.17", - "@angular/compiler": "20.3.17", - "@angular/core": "20.3.17", - "@angular/forms": "20.3.17", + "@angular/common": "20.3.18", + "@angular/compiler": "20.3.18", + "@angular/core": "20.3.18", + "@angular/forms": "20.3.18", "@angular/material": "20.2.14", - "@angular/platform-browser": "20.3.17", - "@angular/platform-browser-dynamic": "20.3.17", - "@angular/router": "20.3.17", + "@angular/platform-browser": "20.3.18", + "@angular/platform-browser-dynamic": "20.3.18", + "@angular/router": "20.3.18", "@auth0/angular-jwt": "^5.2.0", "@flowjs/flow.js": "^2.14.1", "@flowjs/ngx-flow": "20.0.2", @@ -94,13 +94,13 @@ }, "devDependencies": { "@angular-builders/custom-esbuild": "20.0.0", - "@angular-devkit/build-angular": "20.3.18", - "@angular-devkit/core": "20.3.18", - "@angular-devkit/schematics": "20.3.18", - "@angular/build": "20.3.18", - "@angular/cli": "20.3.18", - "@angular/compiler-cli": "20.3.17", - "@angular/language-service": "20.3.17", + "@angular-devkit/build-angular": "20.3.20", + "@angular-devkit/core": "20.3.20", + "@angular-devkit/schematics": "20.3.20", + "@angular/build": "20.3.20", + "@angular/cli": "20.3.20", + "@angular/compiler-cli": "20.3.18", + "@angular/language-service": "20.3.18", "@types/ace-diff": "^2.1.4", "@types/canvas-gauges": "^2.1.8", "@types/flot": "^0.0.36", @@ -121,7 +121,7 @@ "angular-eslint": "~20.7.0", "autoprefixer": "^10.4.23", "directory-tree": "^3.5.2", - "eslint": "9.39.3", + "eslint": "9.39.4", "eslint-plugin-import": "^2.32.0", "eslint-plugin-jsdoc": "^62.4.1", "eslint-plugin-prefer-arrow": "^1.2.3", diff --git a/ui-ngx/patches/@angular+build+20.3.18.patch b/ui-ngx/patches/@angular+build+20.3.20.patch similarity index 100% rename from ui-ngx/patches/@angular+build+20.3.18.patch rename to ui-ngx/patches/@angular+build+20.3.20.patch diff --git a/ui-ngx/patches/@angular+core+20.3.17.patch b/ui-ngx/patches/@angular+core+20.3.18.patch similarity index 92% rename from ui-ngx/patches/@angular+core+20.3.17.patch rename to ui-ngx/patches/@angular+core+20.3.18.patch index aa8fc928ba..12ceb3739d 100644 --- a/ui-ngx/patches/@angular+core+20.3.17.patch +++ b/ui-ngx/patches/@angular+core+20.3.18.patch @@ -1,8 +1,8 @@ diff --git a/node_modules/@angular/core/fesm2022/debug_node.mjs b/node_modules/@angular/core/fesm2022/debug_node.mjs -index d9be60f..24891ce 100755 +index 35c61af..d89462b 100755 --- a/node_modules/@angular/core/fesm2022/debug_node.mjs +++ b/node_modules/@angular/core/fesm2022/debug_node.mjs -@@ -9421,13 +9421,13 @@ function findDirectiveDefMatches(tView, tNode) { +@@ -9428,13 +9428,13 @@ function findDirectiveDefMatches(tView, tNode) { if (isNodeMatchingSelectorList(tNode, def.selectors, /* isProjectionMode */ false)) { matches ??= []; if (isComponentDef(def)) { diff --git a/ui-ngx/pom.xml b/ui-ngx/pom.xml index 760addb451..7cbd38e648 100644 --- a/ui-ngx/pom.xml +++ b/ui-ngx/pom.xml @@ -20,7 +20,7 @@ 4.0.0 org.thingsboard - 4.2.2-SNAPSHOT + 4.2.2.1-SNAPSHOT thingsboard org.thingsboard diff --git a/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts b/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts index c210c54a0b..9d48e3b2ab 100644 --- a/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts +++ b/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts @@ -125,10 +125,6 @@ export class GlobalHttpInterceptor implements HttpInterceptor { this.showError(req.method + ': ' + req.url + '
' + errorResponse.status + ': ' + errorResponse.statusText); } - } else if (errorResponse.status === 504) { - if (!ignoreErrors) { - this.showError('Request timeout'); - } } else { unhandled = true; } diff --git a/ui-ngx/src/app/core/utils.ts b/ui-ngx/src/app/core/utils.ts index 4c7a241931..348d4e7e9b 100644 --- a/ui-ngx/src/app/core/utils.ts +++ b/ui-ngx/src/app/core/utils.ts @@ -23,7 +23,7 @@ import { NULL_UUID } from '@shared/models/id/has-uuid'; import { baseDetailsPageByEntityType, EntityType } from '@shared/models/entity-type.models'; import { HttpClient, HttpErrorResponse } from '@angular/common/http'; import { TranslateService } from '@ngx-translate/core'; -import { serverErrorCodesTranslations } from '@shared/models/constants'; +import { httpStatusMessageMap, serverErrorCodesTranslations } from '@shared/models/constants'; import { SubscriptionEntityInfo } from '@core/api/widget-api.models'; import { CompiledTbFunction, @@ -825,11 +825,13 @@ export function parseHttpErrorMessage(errorResponse: HttpErrorResponse, } else { error = errorResponse.error; } - if (error && !error.message) { - errorMessage = prepareMessageFromData(error); - } else if (error && error.message) { + if (error && error.message) { errorMessage = error.message; timeout = error.timeout ? error.timeout : 0; + } else if (isProxyError(errorResponse)) { + errorMessage = httpStatusMessageMap.get(errorResponse.status); + } else if (error) { + errorMessage = prepareMessageFromData(error); } else { errorMessage = `Unhandled error code ${error ? error.status : '\'Unknown\''}`; } @@ -866,6 +868,14 @@ function prepareMessageFromData(data): string { } } +function isProxyError(errorResponse: HttpErrorResponse): boolean { + if (!httpStatusMessageMap.has(errorResponse.status)) { + return false; + } + const error = errorResponse.error; + return !error || typeof error === 'string' || (typeof error === 'object' && !error.message); +} + export const genNextLabel = (name: string, datasources: Datasource[]): string => { let label = name; let i = 1; diff --git a/ui-ngx/src/app/core/ws/websocket.service.ts b/ui-ngx/src/app/core/ws/websocket.service.ts index ea0884489f..2d7da5ae67 100644 --- a/ui-ngx/src/app/core/ws/websocket.service.ts +++ b/ui-ngx/src/app/core/ws/websocket.service.ts @@ -29,9 +29,11 @@ import { WebsocketDataMsg } from '@shared/models/telemetry/telemetry.models'; import { ActionNotificationShow } from '@core/notification/notification.actions'; +import { NotificationType } from '@core/notification/notification.models'; import Timeout = NodeJS.Timeout; const RECONNECT_INTERVAL = 2000; +const MAX_RECONNECT_INTERVAL = 60000; const WS_IDLE_TIMEOUT = 90000; const MAX_PUBLISH_COMMANDS = 10; @@ -57,6 +59,16 @@ export abstract class WebsocketService implements WsServ errorName = 'WebSocket Error'; + // Exponential backoff: tracks the number of consecutive failed reconnect attempts. + // Reset only after a productive connection (i.e. at least one message received). + // This prevents the open→immediately-closed cycle from resetting the counter. + private reconnectAttempts = 0; + + // Stores the last close-event error code shown to the user during a reconnect cycle. + // Only suppresses duplicate notifications for the same error code; a new error code is still shown. + // Cleared after receiving a successful message. + private lastShownCloseCode: number | null = null; + protected constructor(protected store: Store, protected authService: AuthService, protected ngZone: NgZone, @@ -126,6 +138,8 @@ export abstract class WebsocketService implements WsServ this.subscribersCount = 0; this.cmdWrapper.clear(); if (close) { + this.reconnectAttempts = 0; + this.lastShownCloseCode = null; this.closeSocket(); } } @@ -221,6 +235,10 @@ export abstract class WebsocketService implements WsServ this.processOnMessage(message as WebsocketDataMsg); } this.checkToClose(); + if (this.reconnectAttempts) { + this.reconnectAttempts = 0; + this.lastShownCloseCode = null; + } } private onError(errorEvent) { @@ -231,8 +249,13 @@ export abstract class WebsocketService implements WsServ } private onClose(closeEvent: CloseEvent) { + // Show error notification only when the error code changes to prevent notification spam, + // while still surfacing new, potentially actionable errors during a reconnect cycle. + // lastShownCloseCode is cleared only after a productive connection (onMessage). if (closeEvent && closeEvent.code > 1001 && closeEvent.code !== 1006 - && closeEvent.code !== 1011 && closeEvent.code !== 1012 && closeEvent.code !== 4500) { + && closeEvent.code !== 1011 && closeEvent.code !== 1012 && closeEvent.code !== 4500 + && this.lastShownCloseCode !== closeEvent.code) { + this.lastShownCloseCode = closeEvent.code; this.showWsError(closeEvent.code, closeEvent.reason); } this.isOpening = false; @@ -251,18 +274,28 @@ export abstract class WebsocketService implements WsServ if (this.reconnectTimer) { clearTimeout(this.reconnectTimer); } - this.reconnectTimer = setTimeout(() => this.tryOpenSocket(), RECONNECT_INTERVAL); + const delay = Math.min(RECONNECT_INTERVAL * Math.pow(2, this.reconnectAttempts), MAX_RECONNECT_INTERVAL); + this.reconnectAttempts = Math.min(this.reconnectAttempts + 1, 10); + this.reconnectTimer = setTimeout(() => this.tryOpenSocket(), delay); } } private showWsError(errorCode: number, errorMsg: string) { let message = errorMsg; - if (!message) { - message += `${this.errorName}: error code - ${errorCode}.`; + let notificationType: NotificationType = 'error'; + + if (errorCode === 1008 || (errorMsg && errorMsg.includes('limit reached'))) { + message = 'Too many active sessions. Please close unused browser tabs or sign out from other devices'; + notificationType = 'warn'; + } else if (errorCode === 1009) { + message = 'Too much data to display. Please refresh the page or narrow your request.'; + notificationType = 'warn'; + } else if (!message) { + message = `${this.errorName}: error code - ${errorCode}.`; } - this.store.dispatch(new ActionNotificationShow( - { - message, type: 'error' - })); + + this.store.dispatch(new ActionNotificationShow({ + message, type: notificationType + })); } } diff --git a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html index 6a797a37aa..6d758b8c7b 100644 --- a/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html +++ b/ui-ngx/src/app/modules/home/components/attribute/attribute-table.component.html @@ -93,7 +93,8 @@ (click)="deleteTelemetry($event)"> delete -