From 4bf34281f0810ecf5974b0b8691f3543ec6e638c Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 27 Aug 2024 17:55:41 +0300 Subject: [PATCH 1/6] HttpClient - fixed validation of hostname and port in case proxy usage --- .../org/thingsboard/rule/engine/rest/TbHttpClient.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 96ec648941..8c7dac0b50 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -336,8 +336,8 @@ public class TbHttpClient { String hostname = properties.getProperty(hostProperty); int port = Integer.parseInt(properties.getProperty(portProperty)); - checkProxyHost(config.getProxyHost()); - checkProxyPort(config.getProxyPort()); + checkProxyHost(hostname); + checkProxyPort(port); var proxy = option .type(ProxyProvider.Proxy.HTTP) @@ -362,8 +362,8 @@ public class TbHttpClient { ProxyProvider.Proxy type = SOCKS_VERSION_5.equals(version) ? ProxyProvider.Proxy.SOCKS5 : ProxyProvider.Proxy.SOCKS4; int port = Integer.parseInt(properties.getProperty(SOCKS_PROXY_PORT)); - checkProxyHost(config.getProxyHost()); - checkProxyPort(config.getProxyPort()); + checkProxyHost(hostname); + checkProxyPort(port); ProxyProvider.Builder proxy = option .type(type) From c38584e79f55ba355f0743adf197067a36ab7857 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 27 Aug 2024 18:04:20 +0300 Subject: [PATCH 2/6] Added new config for TbRestApiCallNodeConfiguration - maxInMemoryBufferSizeInKb --- .../java/org/thingsboard/rule/engine/rest/TbHttpClient.java | 1 + .../rule/engine/rest/TbRestApiCallNodeConfiguration.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 8c7dac0b50..d71cc7f021 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -132,6 +132,7 @@ public class TbHttpClient { this.webClient = WebClient.builder() .clientConnector(new ReactorClientHttpConnector(httpClient)) .defaultHeader(HttpHeaders.CONNECTION, "close") //In previous realization this header was present! (Added for hotfix "Connection reset") + .codecs(clientCodecConfigurer -> clientCodecConfigurer.defaultCodecs().maxInMemorySize(config.getMaxInMemoryBufferSizeInKb() * 1024)) .build(); } catch (SSLException e) { throw new TbNodeException(e); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeConfiguration.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeConfiguration.java index 7d2ff7167d..1825bd1ae7 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeConfiguration.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbRestApiCallNodeConfiguration.java @@ -46,6 +46,7 @@ public class TbRestApiCallNodeConfiguration implements NodeConfiguration Date: Tue, 27 Aug 2024 19:22:49 +0300 Subject: [PATCH 3/6] TbHttpClient: Add cause to metadata in cause of buffer overloaded exception --- .../java/org/thingsboard/rule/engine/rest/TbHttpClient.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index d71cc7f021..4830761110 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -132,7 +132,7 @@ public class TbHttpClient { this.webClient = WebClient.builder() .clientConnector(new ReactorClientHttpConnector(httpClient)) .defaultHeader(HttpHeaders.CONNECTION, "close") //In previous realization this header was present! (Added for hotfix "Connection reset") - .codecs(clientCodecConfigurer -> clientCodecConfigurer.defaultCodecs().maxInMemorySize(config.getMaxInMemoryBufferSizeInKb() * 1024)) + .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(config.getMaxInMemoryBufferSizeInKb() * 1024)) .build(); } catch (SSLException e) { throw new TbNodeException(e); @@ -286,6 +286,9 @@ public class TbHttpClient { metaData.putValue(STATUS, restClientResponseException.getStatusText()); metaData.putValue(STATUS_CODE, restClientResponseException.getStatusCode().value() + ""); metaData.putValue(ERROR_BODY, restClientResponseException.getResponseBodyAsString()); + if (restClientResponseException.getStatusCode().is2xxSuccessful()) { + metaData.putValue(ERROR, metaData.getValue(ERROR) + ". Cause: " +restClientResponseException.getCause().getMessage()); + } } return TbMsg.transformMsgMetadata(origMsg, metaData); } From 9593f97582e292b78ad6de11c766a88e9936ef6f Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 29 Aug 2024 17:05:53 +0300 Subject: [PATCH 4/6] TbHttpClient: use default 256 KB for maxInMemorySize in case 0 set in configuration --- .../java/org/thingsboard/rule/engine/rest/TbHttpClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 4830761110..104e477346 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -132,7 +132,8 @@ public class TbHttpClient { this.webClient = WebClient.builder() .clientConnector(new ReactorClientHttpConnector(httpClient)) .defaultHeader(HttpHeaders.CONNECTION, "close") //In previous realization this header was present! (Added for hotfix "Connection reset") - .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(config.getMaxInMemoryBufferSizeInKb() * 1024)) + .codecs(configurer -> configurer.defaultCodecs().maxInMemorySize( + (config.getMaxInMemoryBufferSizeInKb() > 0 ? config.getMaxInMemoryBufferSizeInKb() : 256) * 1024)) .build(); } catch (SSLException e) { throw new TbNodeException(e); From f9c2aa3581c123d6c549bafc5372ef60370ef960 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 30 Aug 2024 10:18:40 +0300 Subject: [PATCH 5/6] TbHttpClient: return cause instead of original exception in case 2xx status code --- .../rule/engine/rest/TbHttpClient.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 104e477346..839a7633fc 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -209,13 +209,23 @@ public class TbHttpClient { semaphore.release(); } - onFailure.accept(processException(msg, throwable), throwable); + onFailure.accept(processException(msg, throwable), processThrowable(throwable)); }); } catch (InterruptedException e) { log.warn("Timeout during waiting for reply!", e); } } + private Throwable processThrowable(Throwable origin) { + if (origin instanceof WebClientResponseException restClientResponseException + && restClientResponseException.getStatusCode().is2xxSuccessful()) { + // return cause instead of original exception in case 2xx status code + // this will provide meaningful error message to the user + return new RuntimeException(restClientResponseException.getCause()); + } + return origin; + } + public URI buildEncodedUri(String endpointUrl) { if (endpointUrl == null) { throw new RuntimeException("Url string cannot be null!"); @@ -287,9 +297,6 @@ public class TbHttpClient { metaData.putValue(STATUS, restClientResponseException.getStatusText()); metaData.putValue(STATUS_CODE, restClientResponseException.getStatusCode().value() + ""); metaData.putValue(ERROR_BODY, restClientResponseException.getResponseBodyAsString()); - if (restClientResponseException.getStatusCode().is2xxSuccessful()) { - metaData.putValue(ERROR, metaData.getValue(ERROR) + ". Cause: " +restClientResponseException.getCause().getMessage()); - } } return TbMsg.transformMsgMetadata(origMsg, metaData); } From b226ebdccfed54a99fa725b39002a0b3694c5644 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 5 Sep 2024 15:15:32 +0300 Subject: [PATCH 6/6] TbHttpClient - added check for maxInMemoryBufferSizeInKb; should be less than system configured maxInMemoryBufferSizeInKb --- .../rule/engine/rest/TbHttpClient.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 839a7633fc..c21914577f 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -80,6 +80,8 @@ public class TbHttpClient { public static final String PROXY_USER = "tb.proxy.user"; public static final String PROXY_PASSWORD = "tb.proxy.password"; + public static final String MAX_IN_MEMORY_BUFFER_SIZE_IN_KB = "tb.http.maxInMemoryBufferSizeInKb"; + private final TbRestApiCallNodeConfiguration config; private EventLoopGroup eventLoopGroup; @@ -129,6 +131,8 @@ public class TbHttpClient { httpClient = httpClient.secure(t -> t.sslContext(sslContext)); } + validateMaxInMemoryBufferSize(config); + this.webClient = WebClient.builder() .clientConnector(new ReactorClientHttpConnector(httpClient)) .defaultHeader(HttpHeaders.CONNECTION, "close") //In previous realization this header was present! (Added for hotfix "Connection reset") @@ -140,6 +144,21 @@ public class TbHttpClient { } } + private void validateMaxInMemoryBufferSize(TbRestApiCallNodeConfiguration config) throws TbNodeException { + int systemMaxInMemoryBufferSizeInKb = 25000; + try { + Properties properties = System.getProperties(); + if (properties.containsKey(MAX_IN_MEMORY_BUFFER_SIZE_IN_KB)) { + systemMaxInMemoryBufferSizeInKb = Integer.parseInt(properties.getProperty(MAX_IN_MEMORY_BUFFER_SIZE_IN_KB)); + } + } catch (Exception ignored) {} + if (config.getMaxInMemoryBufferSizeInKb() > systemMaxInMemoryBufferSizeInKb) { + throw new TbNodeException("The configured maximum in-memory buffer size (in KB) exceeds the system limit for this parameter.\n" + + "The system limit is " + systemMaxInMemoryBufferSizeInKb + " KB.\n" + + "Please use the system variable '" + MAX_IN_MEMORY_BUFFER_SIZE_IN_KB + "' to override the system limit."); + } + } + EventLoopGroup getSharedOrCreateEventLoopGroup(EventLoopGroup eventLoopGroupShared) { if (eventLoopGroupShared != null) { return eventLoopGroupShared;