From c15c4526aa6f81230ddc7ce48186abd312e540a7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Feb 2023 17:05:01 +0200 Subject: [PATCH 01/32] updated RNG used for reset password token to secure one --- .../server/controller/AuthController.java | 25 +++++++++++++++++++ .../src/main/resources/thingsboard.yml | 3 +++ .../server/dao/user/UserServiceImpl.java | 12 ++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index c73ac1ae66..9062ebda34 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -18,8 +18,10 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.ApplicationEventPublisher; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -41,11 +43,13 @@ import org.thingsboard.server.common.data.edge.EdgeEventActionType; import org.thingsboard.server.common.data.exception.ThingsboardErrorCode; import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent; import org.thingsboard.server.common.data.security.event.UserSessionInvalidationEvent; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; +import org.thingsboard.server.common.msg.tools.TbRateLimits; import org.thingsboard.server.dao.audit.AuditLogService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; @@ -62,6 +66,8 @@ import org.thingsboard.server.service.security.system.SystemSecurityService; import javax.servlet.http.HttpServletRequest; import java.net.URI; import java.net.URISyntaxException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; @RestController @TbCoreComponent @@ -69,6 +75,10 @@ import java.net.URISyntaxException; @Slf4j @RequiredArgsConstructor public class AuthController extends BaseController { + @Value("${rate_limits.reset_password_per_user.configuration:5:300}") + @Getter + private String defaultLimitsConfiguration; + private final ConcurrentMap resetPasswordRateLimits = new ConcurrentHashMap<>(); private final BCryptPasswordEncoder passwordEncoder; private final JwtTokenFactory tokenFactory; private final MailService mailService; @@ -211,6 +221,12 @@ public class AuthController extends BaseController { HttpStatus responseStatus; String resetURI = "/login/resetPassword"; UserCredentials userCredentials = userService.findUserCredentialsByResetToken(TenantId.SYS_TENANT_ID, resetToken); + + TbRateLimits tbRateLimits = getTbRateLimits(userCredentials); + if (!tbRateLimits.tryConsume()) { + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); + } + if (userCredentials != null) { try { URI location = new URI(resetURI + "?resetToken=" + resetToken); @@ -323,4 +339,13 @@ public class AuthController extends BaseController { throw handleException(e); } } + + private TbRateLimits getTbRateLimits(UserCredentials userCredentials) { + TbRateLimits rateLimit = resetPasswordRateLimits.get(userCredentials.getUserId()); + if (rateLimit == null) { + rateLimit = new TbRateLimits(defaultLimitsConfiguration, true); + resetPasswordRateLimits.put(userCredentials.getUserId(), rateLimit); + } + return rateLimit; + } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 5549861950..2f99b285fe 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1209,3 +1209,6 @@ management: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). include: '${METRICS_ENDPOINTS_EXPOSE:info}' +rate_limits: + reset_password_per_user: + configuration: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:5}" diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index c8c5175e5f..e4dd882059 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -47,6 +47,8 @@ import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; +import java.security.SecureRandom; +import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -192,10 +194,18 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic if (!userCredentials.isEnabled()) { throw new DisabledException(String.format("User credentials not enabled [%s]", email)); } - userCredentials.setResetToken(StringUtils.randomAlphanumeric(DEFAULT_TOKEN_LENGTH)); + userCredentials.setResetToken(generateSafeToken()); return saveUserCredentials(tenantId, userCredentials); } + private String generateSafeToken() { + SecureRandom random = new SecureRandom(); + byte[] bytes = new byte[DEFAULT_TOKEN_LENGTH]; + random.nextBytes(bytes); + Base64.Encoder encoder = Base64.getUrlEncoder().withoutPadding(); + return encoder.encodeToString(bytes); + } + @Override public UserCredentials requestExpiredPasswordReset(TenantId tenantId, UserCredentialsId userCredentialsId) { UserCredentials userCredentials = userCredentialsDao.findById(tenantId, userCredentialsId.getId()); From 2c5c0a17bfdd3c2afe6000b443b56dd32ae8c87c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Feb 2023 17:12:45 +0200 Subject: [PATCH 02/32] refactoring --- .../server/common/data/StringUtils.java | 11 +++++++++++ .../server/dao/user/UserServiceImpl.java | 19 ++++--------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 6a2b0a58d6..7a4c180b9c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -18,6 +18,9 @@ package org.thingsboard.server.common.data; import com.google.common.base.Splitter; import org.apache.commons.lang3.RandomStringUtils; +import java.security.SecureRandom; +import java.util.Base64; + import static org.apache.commons.lang3.StringUtils.repeat; public class StringUtils { @@ -180,4 +183,12 @@ public class StringUtils { return RandomStringUtils.randomAlphabetic(count); } + public static String generateSafeToken(int length) { + SecureRandom random = new SecureRandom(); + byte[] bytes = new byte[length]; + random.nextBytes(bytes); + Base64.Encoder encoder = Base64.getUrlEncoder().withoutPadding(); + return encoder.encodeToString(bytes); + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java index e4dd882059..b4ad62c0fc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java +++ b/dao/src/main/java/org/thingsboard/server/dao/user/UserServiceImpl.java @@ -29,7 +29,6 @@ import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; @@ -40,19 +39,17 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.UserCredentials; -import org.thingsboard.server.common.data.security.UserSettings; import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent; import org.thingsboard.server.dao.entity.AbstractEntityService; import org.thingsboard.server.dao.exception.IncorrectParameterException; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.service.PaginatedRemover; -import java.security.SecureRandom; -import java.util.Base64; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import static org.thingsboard.server.common.data.StringUtils.generateSafeToken; import static org.thingsboard.server.dao.service.Validator.validateId; import static org.thingsboard.server.dao.service.Validator.validatePageLink; import static org.thingsboard.server.dao.service.Validator.validateString; @@ -128,7 +125,7 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic if (user.getId() == null) { UserCredentials userCredentials = new UserCredentials(); userCredentials.setEnabled(false); - userCredentials.setActivateToken(StringUtils.randomAlphanumeric(DEFAULT_TOKEN_LENGTH)); + userCredentials.setActivateToken(generateSafeToken(DEFAULT_TOKEN_LENGTH)); userCredentials.setUserId(new UserId(savedUser.getUuidId())); saveUserCredentialsAndPasswordHistory(user.getTenantId(), userCredentials); } @@ -194,25 +191,17 @@ public class UserServiceImpl extends AbstractEntityService implements UserServic if (!userCredentials.isEnabled()) { throw new DisabledException(String.format("User credentials not enabled [%s]", email)); } - userCredentials.setResetToken(generateSafeToken()); + userCredentials.setResetToken(generateSafeToken(DEFAULT_TOKEN_LENGTH)); return saveUserCredentials(tenantId, userCredentials); } - private String generateSafeToken() { - SecureRandom random = new SecureRandom(); - byte[] bytes = new byte[DEFAULT_TOKEN_LENGTH]; - random.nextBytes(bytes); - Base64.Encoder encoder = Base64.getUrlEncoder().withoutPadding(); - return encoder.encodeToString(bytes); - } - @Override public UserCredentials requestExpiredPasswordReset(TenantId tenantId, UserCredentialsId userCredentialsId) { UserCredentials userCredentials = userCredentialsDao.findById(tenantId, userCredentialsId.getId()); if (!userCredentials.isEnabled()) { throw new IncorrectParameterException("Unable to reset password for inactive user"); } - userCredentials.setResetToken(StringUtils.randomAlphanumeric(DEFAULT_TOKEN_LENGTH)); + userCredentials.setResetToken(generateSafeToken(DEFAULT_TOKEN_LENGTH)); return saveUserCredentials(tenantId, userCredentials); } From 644f94d01288fc4c2a9411a874ae066a3e208742 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Wed, 15 Feb 2023 17:14:48 +0200 Subject: [PATCH 03/32] refactoring --- .../java/org/thingsboard/server/controller/AuthController.java | 2 +- application/src/main/resources/thingsboard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 9062ebda34..336d3f8353 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -75,7 +75,7 @@ import java.util.concurrent.ConcurrentMap; @Slf4j @RequiredArgsConstructor public class AuthController extends BaseController { - @Value("${rate_limits.reset_password_per_user.configuration:5:300}") + @Value("${rate_limits.reset_password_per_user.configuration:5:3600}") @Getter private String defaultLimitsConfiguration; private final ConcurrentMap resetPasswordRateLimits = new ConcurrentHashMap<>(); diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 2f99b285fe..7e08477419 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1211,4 +1211,4 @@ management: include: '${METRICS_ENDPOINTS_EXPOSE:info}' rate_limits: reset_password_per_user: - configuration: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:5}" + configuration: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:3600}" From 314071b7615a3b7435fdb070fca3f6bbfd212c41 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 13 Feb 2023 14:22:32 +0100 Subject: [PATCH 04/32] SPRING_DATASOURCE_HIKARI_REGISTER_MBEANS parameter added --- application/src/main/resources/thingsboard.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 451377c3b0..ecd624172c 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -566,6 +566,7 @@ spring: password: "${SPRING_DATASOURCE_PASSWORD:postgres}" hikari: maximumPoolSize: "${SPRING_DATASOURCE_MAXIMUM_POOL_SIZE:16}" + registerMbeans: "${SPRING_DATASOURCE_HIKARI_REGISTER_MBEANS:false}" # true - enable MBean to diagnose pools state via JMX # Audit log parameters audit-log: From ab643063f253eb018a7f600a47035532a7b041ec Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 16 Feb 2023 11:19:42 +0200 Subject: [PATCH 05/32] refactoring --- .../server/controller/AuthController.java | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 336d3f8353..88de329883 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -18,7 +18,6 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.databind.node.ObjectNode; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; @@ -75,8 +74,8 @@ import java.util.concurrent.ConcurrentMap; @Slf4j @RequiredArgsConstructor public class AuthController extends BaseController { + @Value("${rate_limits.reset_password_per_user.configuration:5:3600}") - @Getter private String defaultLimitsConfiguration; private final ConcurrentMap resetPasswordRateLimits = new ConcurrentHashMap<>(); private final BCryptPasswordEncoder passwordEncoder; @@ -222,12 +221,11 @@ public class AuthController extends BaseController { String resetURI = "/login/resetPassword"; UserCredentials userCredentials = userService.findUserCredentialsByResetToken(TenantId.SYS_TENANT_ID, resetToken); - TbRateLimits tbRateLimits = getTbRateLimits(userCredentials); - if (!tbRateLimits.tryConsume()) { - return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); - } - if (userCredentials != null) { + TbRateLimits tbRateLimits = getTbRateLimits(userCredentials.getUserId()); + if (!tbRateLimits.tryConsume()) { + return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS).build(); + } try { URI location = new URI(resetURI + "?resetToken=" + resetToken); headers.setLocation(location); @@ -340,12 +338,8 @@ public class AuthController extends BaseController { } } - private TbRateLimits getTbRateLimits(UserCredentials userCredentials) { - TbRateLimits rateLimit = resetPasswordRateLimits.get(userCredentials.getUserId()); - if (rateLimit == null) { - rateLimit = new TbRateLimits(defaultLimitsConfiguration, true); - resetPasswordRateLimits.put(userCredentials.getUserId(), rateLimit); - } - return rateLimit; + private TbRateLimits getTbRateLimits(UserId userId) { + return resetPasswordRateLimits.computeIfAbsent(userId, + key -> new TbRateLimits(defaultLimitsConfiguration, true)); } } From 831018af40795f7908b66a1ea2ff8b8c09d01160 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 17 Feb 2023 14:41:20 +0200 Subject: [PATCH 06/32] minor refactoring --- .../java/org/thingsboard/server/controller/AuthController.java | 2 +- application/src/main/resources/thingsboard.yml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 88de329883..d0e382f888 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -75,7 +75,7 @@ import java.util.concurrent.ConcurrentMap; @RequiredArgsConstructor public class AuthController extends BaseController { - @Value("${rate_limits.reset_password_per_user.configuration:5:3600}") + @Value("${rate_limits.reset_password_per_user:5:3600}") private String defaultLimitsConfiguration; private final ConcurrentMap resetPasswordRateLimits = new ConcurrentHashMap<>(); private final BCryptPasswordEncoder passwordEncoder; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7e08477419..7a5a1e57fd 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -1210,5 +1210,4 @@ management: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). include: '${METRICS_ENDPOINTS_EXPOSE:info}' rate_limits: - reset_password_per_user: - configuration: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:3600}" + reset_password_per_user: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:3600}" From 935f5b806d426a8830801ed988f2d1b21056d779 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Mon, 20 Feb 2023 17:01:42 +0200 Subject: [PATCH 07/32] fix issue with remove latest when interval is set to from x to x + 1 where x value of ts entry --- .../thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java index afffdbb0cc..b58f0dc7c7 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/SqlTimeseriesLatestDao.java @@ -228,7 +228,7 @@ public class SqlTimeseriesLatestDao extends BaseAbstractSqlTimeseriesDao impleme long ts = latest.getTs(); ListenableFuture removedLatestFuture; - if (ts > query.getStartTs() && ts <= query.getEndTs()) { + if (ts >= query.getStartTs() && ts < query.getEndTs()) { TsKvLatestEntity latestEntity = new TsKvLatestEntity(); latestEntity.setEntityId(entityId.getId()); latestEntity.setKey(getOrSaveKeyId(query.getKey())); From 9bbb88ab0fa244c2a84b6deffc9116b2fa8642c5 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 21 Feb 2023 16:50:09 +0200 Subject: [PATCH 08/32] Add functionality to support multiple certificate entries in pem file --- .../credentials/CertPemCredentials.java | 46 +++++--- .../credentials/CertPemCredentialsTest.java | 58 ++++++++++ .../src/test/resources/pem/empty.pem | 0 .../src/test/resources/pem/tb-cloud-chain.pem | 103 ++++++++++++++++++ .../src/test/resources/pem/tb-cloud.pem | 32 ++++++ 5 files changed, 225 insertions(+), 14 deletions(-) create mode 100644 rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/empty.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem create mode 100644 rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 48d71873eb..480c4a0689 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -57,6 +57,10 @@ import java.security.cert.X509Certificate; import java.security.spec.KeySpec; import java.security.spec.PKCS8EncodedKeySpec; import java.security.spec.RSAPrivateCrtKeySpec; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -104,7 +108,7 @@ public class CertPemCredentials implements ClientCredentials { } private KeyManagerFactory createAndInitKeyManagerFactory() throws Exception { - X509Certificate certHolder = readCertFile(cert); + List certHolders = readCertFile(cert); Object keyObject = readPrivateKeyFile(privateKey); char[] passwordCharArray = "".toCharArray(); if (!StringUtils.isEmpty(password)) { @@ -129,43 +133,57 @@ public class CertPemCredentials implements ClientCredentials { KeyStore clientKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); clientKeyStore.load(null, null); - clientKeyStore.setCertificateEntry("cert", certHolder); + for (X509Certificate certHolder : certHolders) { + clientKeyStore.setCertificateEntry(createCertEntryAlias("cert-", certHolder), certHolder); + } clientKeyStore.setKeyEntry("private-key", privateKey, passwordCharArray, - new Certificate[]{certHolder}); - + certHolders.toArray(new Certificate[]{})); KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keyManagerFactory.init(clientKeyStore, passwordCharArray); return keyManagerFactory; } protected TrustManagerFactory createAndInitTrustManagerFactory() throws Exception { - X509Certificate caCertHolder; - caCertHolder = readCertFile(caCert); + List caCertHolders = readCertFile(caCert); KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); - caKeyStore.setCertificateEntry("caCert-cert", caCertHolder); + for (X509Certificate caCertHolder : caCertHolders) { + caKeyStore.setCertificateEntry(createCertEntryAlias("caCert-cert-", caCertHolder), caCertHolder); + } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); trustManagerFactory.init(caKeyStore); return trustManagerFactory; } - private X509Certificate readCertFile(String fileContent) throws Exception { - X509Certificate certificate = null; - if (fileContent != null && !fileContent.trim().isEmpty()) { - fileContent = fileContent.replace("-----BEGIN CERTIFICATE-----", "") + private String createCertEntryAlias(String prefix, X509Certificate certificate) { + return prefix + "-" + certificate.getSubjectDN().getName(); + } + + List readCertFile(String fileContent) throws Exception { + if (fileContent == null || fileContent.trim().isEmpty()) { + return Collections.emptyList(); + } + + List certificates = new ArrayList<>(); + String[] pems = fileContent.trim().split("-----END CERTIFICATE-----"); + for (String pem : pems) { + if (pem.trim().isEmpty()) { + continue; + } + pem = pem.replace("-----BEGIN CERTIFICATE-----", "") .replace("-----END CERTIFICATE-----", "") .replaceAll("\\s", ""); - byte[] decoded = Base64.decodeBase64(fileContent); + byte[] decoded = Base64.decodeBase64(pem); CertificateFactory certFactory = CertificateFactory.getInstance("X.509"); try (InputStream inStream = new ByteArrayInputStream(decoded)) { - certificate = (X509Certificate) certFactory.generateCertificate(inStream); + certificates.add((X509Certificate) certFactory.generateCertificate(inStream)); } } - return certificate; + return certificates; } private PrivateKey readPrivateKeyFile(String fileContent) throws Exception { diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java new file mode 100644 index 0000000000..e8126cade5 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -0,0 +1,58 @@ +package org.thingsboard.rule.engine.credentials; + +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.security.cert.X509Certificate; +import java.util.List; + +public class CertPemCredentialsTest { + + private final CertPemCredentials credentials = new CertPemCredentials(); + + @Test + public void testChainOfCertificates() throws Exception { + String fileContent = fileContent("pem/tb-cloud-chain.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(4, x509Certificates.size()); + Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + x509Certificates.get(0).getSubjectDN().getName()); + Assert.assertEquals("CN=Sectigo ECC Organization Validation Secure Server CA, O=Sectigo Limited, L=Salford, ST=Greater Manchester, C=GB", + x509Certificates.get(1).getSubjectDN().getName()); + Assert.assertEquals("CN=USERTrust ECC Certification Authority, O=The USERTRUST Network, L=Jersey City, ST=New Jersey, C=US", + x509Certificates.get(2).getSubjectDN().getName()); + Assert.assertEquals("CN=AAA Certificate Services, O=Comodo CA Limited, L=Salford, ST=Greater Manchester, C=GB", + x509Certificates.get(3).getSubjectDN().getName()); + } + + @Test + public void testSingleCertificate() throws Exception { + String fileContent = fileContent("pem/tb-cloud.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(1, x509Certificates.size()); + Assert.assertEquals("CN=*.thingsboard.cloud, O=\"ThingsBoard, Inc.\", ST=New York, C=US", + x509Certificates.get(0).getSubjectDN().getName()); + } + + @Test + public void testEmptyFileContent() throws Exception { + String fileContent = fileContent("pem/empty.pem"); + + List x509Certificates = credentials.readCertFile(fileContent); + + Assert.assertEquals(0, x509Certificates.size()); + } + + private String fileContent(String fileName) throws IOException { + ClassLoader classLoader = getClass().getClassLoader(); + File file = new File(classLoader.getResource(fileName).getFile()); + return FileUtils.readFileToString(file, "UTF-8"); + } +} \ No newline at end of file diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/empty.pem b/rule-engine/rule-engine-components/src/test/resources/pem/empty.pem new file mode 100644 index 0000000000..e69de29bb2 diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem new file mode 100644 index 0000000000..23049273c7 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud-chain.pem @@ -0,0 +1,103 @@ +-----BEGIN CERTIFICATE----- +MIIFejCCBSCgAwIBAgIQT2YV5NVp2PAY1O5rxMlj6DAKBggqhkjOPQQDAjCBlTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRT +ZWN0aWdvIEVDQyBPcmdhbml6YXRpb24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVy +IENBMB4XDTIxMTAwNTAwMDAwMFoXDTIyMTAwNTIzNTk1OVowWjELMAkGA1UEBhMC +VVMxETAPBgNVBAgTCE5ldyBZb3JrMRowGAYDVQQKExFUaGluZ3NCb2FyZCwgSW5j +LjEcMBoGA1UEAwwTKi50aGluZ3Nib2FyZC5jbG91ZDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABNkK/UerQZXPP0H/Tl8YhRPlzW85yTAcQ5hXhs2fyXn7Bdj4EueQuZrv +Pw98xwHJr87jslFbS/WiSdtBYPvjsUyXqh7aMvOcEhSgEOWDmtoj3P1Xk1hNLb6m +xAQfFL8cZ6OCA20wggNpMB8GA1UdIwQYMBaAFE1K78RGsxKtT06asVniUasIEHgI +MB0GA1UdDgQWBBRr4HG23dsao68r9r7obGxB70ptBzAOBgNVHQ8BAf8EBAMCB4Aw +DAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwSgYD +VR0gBEMwQTA1BgwrBgEEAbIxAQIBAwQwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9z +ZWN0aWdvLmNvbS9DUFMwCAYGZ4EMAQICMFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6 +Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb0VDQ09yZ2FuaXphdGlvblZhbGlkYXRp +b25TZWN1cmVTZXJ2ZXJDQS5jcmwwgYoGCCsGAQUFBwEBBH4wfDBVBggrBgEFBQcw +AoZJaHR0cDovL2NydC5zZWN0aWdvLmNvbS9TZWN0aWdvRUNDT3JnYW5pemF0aW9u +VmFsaWRhdGlvblNlY3VyZVNlcnZlckNBLmNydDAjBggrBgEFBQcwAYYXaHR0cDov +L29jc3Auc2VjdGlnby5jb20wMQYDVR0RBCowKIITKi50aGluZ3Nib2FyZC5jbG91 +ZIIRdGhpbmdzYm9hcmQuY2xvdWQwggGABgorBgEEAdZ5AgQCBIIBcASCAWwBagB2 +AEalVet1+pEgMLWiiWn0830RLEF0vv1JuIWr8vxw/m1HAAABfFDbhtMAAAQDAEcw +RQIhAKNykhkQTngK0yOcYHGHUQSy6JmJYl+5nc1qELirPHwAAiBgV2Db5ZFHNvzn +zp9Ob/OG0o36z6rcilbLI/daZwnyewB3AEHIyrHfIkZKEMahOglCh15OMYsbA+vr +S8do8JBilgb2AAABfFDbhqYAAAQDAEgwRgIhALFvTbapKhO7DPrF6KtE9sjFMMth +qjqaeaHYN6JGnUAIAiEApEW+rxlzxH1+qEwJrFyQLr5rSKTuEoSjv3hbrzb9GQ4A +dwApeb7wnjk5IfBWc59jpXflvld9nGAK+PlNXSZcJV3HhAAAAXxQ24ZnAAAEAwBI +MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL +1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ +GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR +13sr0/ED+2PUom2dnfg= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDrjCCAzOgAwIBAgIQNb50Y4yz6d4oBXC3l4CzZzAKBggqhkjOPQQDAzCBiDEL +MAkGA1UEBhMCVVMxEzARBgNVBAgTCk5ldyBKZXJzZXkxFDASBgNVBAcTC0plcnNl +eSBDaXR5MR4wHAYDVQQKExVUaGUgVVNFUlRSVVNUIE5ldHdvcmsxLjAsBgNVBAMT +JVVTRVJUcnVzdCBFQ0MgQ2VydGlmaWNhdGlvbiBBdXRob3JpdHkwHhcNMTgxMTAy +MDAwMDAwWhcNMzAxMjMxMjM1OTU5WjCBlTELMAkGA1UEBhMCR0IxGzAZBgNVBAgT +EkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UEBxMHU2FsZm9yZDEYMBYGA1UEChMP +U2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRTZWN0aWdvIEVDQyBPcmdhbml6YXRp +b24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVyIENBMFkwEwYHKoZIzj0CAQYIKoZI +zj0DAQcDQgAEnI5cCmFvoVij0NXO+vxE+f+6Bh57FhpyH0LTCrJmzfsPSXIhTSex +r92HOlz+aHqoGE0vSe/CSwLFoWcZ8W1jOaOCAW4wggFqMB8GA1UdIwQYMBaAFDrh +CYbUzxnClnZ0SXbc4DXGY2OaMB0GA1UdDgQWBBRNSu/ERrMSrU9OmrFZ4lGrCBB4 +CDAOBgNVHQ8BAf8EBAMCAYYwEgYDVR0TAQH/BAgwBgEB/wIBADAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwGwYDVR0gBBQwEjAGBgRVHSAAMAgGBmeBDAEC +AjBQBgNVHR8ESTBHMEWgQ6BBhj9odHRwOi8vY3JsLnVzZXJ0cnVzdC5jb20vVVNF +UlRydXN0RUNDQ2VydGlmaWNhdGlvbkF1dGhvcml0eS5jcmwwdgYIKwYBBQUHAQEE +ajBoMD8GCCsGAQUFBzAChjNodHRwOi8vY3J0LnVzZXJ0cnVzdC5jb20vVVNFUlRy +dXN0RUNDQWRkVHJ1c3RDQS5jcnQwJQYIKwYBBQUHMAGGGWh0dHA6Ly9vY3NwLnVz +ZXJ0cnVzdC5jb20wCgYIKoZIzj0EAwMDaQAwZgIxAOk//uo7i/MoeKdcyeqvjOXs +BJFGLI+1i0d+Tty7zEnn2w4DNS21TK8wmY3Kjm3EmQIxAPI1qHM/I+OS+hx0OZhG +fDoNifTe/GxgWZ1gOYQKzn6lwP0yGKlrP+7vrVC8IczJ4A== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIID0zCCArugAwIBAgIQVmcdBOpPmUxvEIFHWdJ1lDANBgkqhkiG9w0BAQwFADB7 +MQswCQYDVQQGEwJHQjEbMBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYD +VQQHDAdTYWxmb3JkMRowGAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UE +AwwYQUFBIENlcnRpZmljYXRlIFNlcnZpY2VzMB4XDTE5MDMxMjAwMDAwMFoXDTI4 +MTIzMTIzNTk1OVowgYgxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpOZXcgSmVyc2V5 +MRQwEgYDVQQHEwtKZXJzZXkgQ2l0eTEeMBwGA1UEChMVVGhlIFVTRVJUUlVTVCBO +ZXR3b3JrMS4wLAYDVQQDEyVVU0VSVHJ1c3QgRUNDIENlcnRpZmljYXRpb24gQXV0 +aG9yaXR5MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEGqxUWqn5aCPnetUkb1PGWthL +q8bVttHmc3Gu3ZzWDGH926CJA7gFFOxXzu5dP+Ihs8731Ip54KODfi2X0GHE8Znc +JZFjq38wo7Rw4sehM5zzvy5cU7Ffs30yf4o043l5o4HyMIHvMB8GA1UdIwQYMBaA +FKARCiM+lvEH7OKvKe+CpX/QMKS0MB0GA1UdDgQWBBQ64QmG1M8ZwpZ2dEl23OA1 +xmNjmjAOBgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zARBgNVHSAECjAI +MAYGBFUdIAAwQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5j +b20vQUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNAYIKwYBBQUHAQEEKDAmMCQG +CCsGAQUFBzABhhhodHRwOi8vb2NzcC5jb21vZG9jYS5jb20wDQYJKoZIhvcNAQEM +BQADggEBABns652JLCALBIAdGN5CmXKZFjK9Dpx1WywV4ilAbe7/ctvbq5AfjJXy +ij0IckKJUAfiORVsAYfZFhr1wHUrxeZWEQff2Ji8fJ8ZOd+LygBkc7xGEJuTI42+ +FsMuCIKchjN0djsoTI0DQoWz4rIjQtUfenVqGtF8qmchxDM6OW1TyaLtYiKou+JV +bJlsQ2uRl9EMC5MCHdK8aXdJ5htN978UeAOwproLtOGFfy/cQjutdAFI3tZs4RmY +CV4Ks2dH/hzg1cEo70qLRDEmBDeNiXQ2Lu+lIg+DdEmSx/cQwgwp+7e9un/jX9Wf +8qn0dNW44bOwgeThpWOjzOoEeJBuv/c= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIEMjCCAxqgAwIBAgIBATANBgkqhkiG9w0BAQUFADB7MQswCQYDVQQGEwJHQjEb +MBkGA1UECAwSR3JlYXRlciBNYW5jaGVzdGVyMRAwDgYDVQQHDAdTYWxmb3JkMRow +GAYDVQQKDBFDb21vZG8gQ0EgTGltaXRlZDEhMB8GA1UEAwwYQUFBIENlcnRpZmlj +YXRlIFNlcnZpY2VzMB4XDTA0MDEwMTAwMDAwMFoXDTI4MTIzMTIzNTk1OVowezEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNVBAMM +GEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBAL5AnfRu4ep2hxxNRUSOvkbIgwadwSr+GB+O5AL686tdUIoWMQua +BtDFcCLNSS1UY8y2bmhGC1Pqy0wkwLxyTurxFa70VJoSCsN6sjNg4tqJVfMiWPPe +3M/vg4aijJRPn2jymJBGhCfHdr/jzDUsi14HZGWCwEiwqJH5YZ92IFCokcdmtet4 +YgNW8IoaE+oxox6gmf049vYnMlhvB/VruPsUK6+3qszWY19zjNoFmag4qMsXeDZR +rOme9Hg6jc8P2ULimAyrL58OAd7vn5lJ8S3frHRNG5i1R8XlKdH5kBjHYpy+g8cm +ez6KJcfA3Z3mNWgQIJ2P2N7Sw4ScDV7oL8kCAwEAAaOBwDCBvTAdBgNVHQ4EFgQU +oBEKIz6W8Qfs4q8p74Klf9AwpLQwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQF +MAMBAf8wewYDVR0fBHQwcjA4oDagNIYyaHR0cDovL2NybC5jb21vZG9jYS5jb20v +QUFBQ2VydGlmaWNhdGVTZXJ2aWNlcy5jcmwwNqA0oDKGMGh0dHA6Ly9jcmwuY29t +b2RvLm5ldC9BQUFDZXJ0aWZpY2F0ZVNlcnZpY2VzLmNybDANBgkqhkiG9w0BAQUF +AAOCAQEACFb8AvCb6P+k+tZ7xkSAzk/ExfYAWMymtrwUSWgEdujm7l3sAg9g1o1Q +GE8mTgHj5rCl7r+8dFRBv/38ErjHT1r0iWAFf2C3BUrz9vHCv8S5dIa2LX1rzNLz +Rt0vxuBqw8M0Ayx9lt1awg6nCpnBBYurDC/zXDrPbDdVCYfeU0BsWO/8tqtlbgT2 +G9w84FoVxp7Z8VlIMCFlA2zs6SFz7JsDoeA3raAVGI/6ugLOpyypEBMs1OUIJqsi +l2D4kF501KKaU73yqWjgom7C12yxow+ev+to51byrvLjKzg6CYG1a4XXvi3tPxq3 +smPi9WIsgtRqAEFQ8TmDn5XpNpaYbg== +-----END CERTIFICATE----- + diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem new file mode 100644 index 0000000000..d75be0bf10 --- /dev/null +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFejCCBSCgAwIBAgIQT2YV5NVp2PAY1O5rxMlj6DAKBggqhkjOPQQDAjCBlTEL +MAkGA1UEBhMCR0IxGzAZBgNVBAgTEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4GA1UE +BxMHU2FsZm9yZDEYMBYGA1UEChMPU2VjdGlnbyBMaW1pdGVkMT0wOwYDVQQDEzRT +ZWN0aWdvIEVDQyBPcmdhbml6YXRpb24gVmFsaWRhdGlvbiBTZWN1cmUgU2VydmVy +IENBMB4XDTIxMTAwNTAwMDAwMFoXDTIyMTAwNTIzNTk1OVowWjELMAkGA1UEBhMC +VVMxETAPBgNVBAgTCE5ldyBZb3JrMRowGAYDVQQKExFUaGluZ3NCb2FyZCwgSW5j +LjEcMBoGA1UEAwwTKi50aGluZ3Nib2FyZC5jbG91ZDB2MBAGByqGSM49AgEGBSuB +BAAiA2IABNkK/UerQZXPP0H/Tl8YhRPlzW85yTAcQ5hXhs2fyXn7Bdj4EueQuZrv +Pw98xwHJr87jslFbS/WiSdtBYPvjsUyXqh7aMvOcEhSgEOWDmtoj3P1Xk1hNLb6m +xAQfFL8cZ6OCA20wggNpMB8GA1UdIwQYMBaAFE1K78RGsxKtT06asVniUasIEHgI +MB0GA1UdDgQWBBRr4HG23dsao68r9r7obGxB70ptBzAOBgNVHQ8BAf8EBAMCB4Aw +DAYDVR0TAQH/BAIwADAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwSgYD +VR0gBEMwQTA1BgwrBgEEAbIxAQIBAwQwJTAjBggrBgEFBQcCARYXaHR0cHM6Ly9z +ZWN0aWdvLmNvbS9DUFMwCAYGZ4EMAQICMFoGA1UdHwRTMFEwT6BNoEuGSWh0dHA6 +Ly9jcmwuc2VjdGlnby5jb20vU2VjdGlnb0VDQ09yZ2FuaXphdGlvblZhbGlkYXRp +b25TZWN1cmVTZXJ2ZXJDQS5jcmwwgYoGCCsGAQUFBwEBBH4wfDBVBggrBgEFBQcw +AoZJaHR0cDovL2NydC5zZWN0aWdvLmNvbS9TZWN0aWdvRUNDT3JnYW5pemF0aW9u +VmFsaWRhdGlvblNlY3VyZVNlcnZlckNBLmNydDAjBggrBgEFBQcwAYYXaHR0cDov +L29jc3Auc2VjdGlnby5jb20wMQYDVR0RBCowKIITKi50aGluZ3Nib2FyZC5jbG91 +ZIIRdGhpbmdzYm9hcmQuY2xvdWQwggGABgorBgEEAdZ5AgQCBIIBcASCAWwBagB2 +AEalVet1+pEgMLWiiWn0830RLEF0vv1JuIWr8vxw/m1HAAABfFDbhtMAAAQDAEcw +RQIhAKNykhkQTngK0yOcYHGHUQSy6JmJYl+5nc1qELirPHwAAiBgV2Db5ZFHNvzn +zp9Ob/OG0o36z6rcilbLI/daZwnyewB3AEHIyrHfIkZKEMahOglCh15OMYsbA+vr +S8do8JBilgb2AAABfFDbhqYAAAQDAEgwRgIhALFvTbapKhO7DPrF6KtE9sjFMMth +qjqaeaHYN6JGnUAIAiEApEW+rxlzxH1+qEwJrFyQLr5rSKTuEoSjv3hbrzb9GQ4A +dwApeb7wnjk5IfBWc59jpXflvld9nGAK+PlNXSZcJV3HhAAAAXxQ24ZnAAAEAwBI +MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL +1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ +GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR +13sr0/ED+2PUom2dnfg= +-----END CERTIFICATE----- \ No newline at end of file From 881cc886f9cd8a5ae99a7260d0cb6e521e247bff Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 21 Feb 2023 16:53:08 +0200 Subject: [PATCH 09/32] Add newline to end of files --- .../rule/engine/credentials/CertPemCredentialsTest.java | 2 +- .../rule-engine-components/src/test/resources/pem/tb-cloud.pem | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index e8126cade5..132b60554b 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -55,4 +55,4 @@ public class CertPemCredentialsTest { File file = new File(classLoader.getResource(fileName).getFile()); return FileUtils.readFileToString(file, "UTF-8"); } -} \ No newline at end of file +} diff --git a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem index d75be0bf10..8aba29c995 100644 --- a/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem +++ b/rule-engine/rule-engine-components/src/test/resources/pem/tb-cloud.pem @@ -29,4 +29,4 @@ MEYCIQDReSpJPzADl/fBdCvyZwWu3Ubi6y0h/S4i7RIjf5L5pAIhAJ1oUmNmRWQL 1U3HAqz2V8ckH/rgA0BR+CkGBigt3dfwMAoGCCqGSM49BAMCA0gAMEUCID0Wv3hJ GyO4kxlCsA/Kruzew8Wr/0k84csyaCo0k16kAiEAtAobCIzx/PIDWU2rX5elBNiR 13sr0/ED+2PUom2dnfg= ------END CERTIFICATE----- \ No newline at end of file +-----END CERTIFICATE----- From 1a6cd82461b306e1ffcb9c83669326f9ad41dc3a Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Feb 2023 13:29:15 +0200 Subject: [PATCH 10/32] License header fix --- .../credentials/CertPemCredentialsTest.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java index 132b60554b..904f470001 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/credentials/CertPemCredentialsTest.java @@ -1,3 +1,18 @@ +/** + * Copyright © 2016-2023 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.thingsboard.rule.engine.credentials; import org.apache.commons.io.FileUtils; From d0fbecc4c5eba4e7fd4d1ffae364e96a05f4b693 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Feb 2023 16:10:53 +0200 Subject: [PATCH 11/32] Organize imports --- .../thingsboard/rule/engine/credentials/CertPemCredentials.java | 1 - 1 file changed, 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 480c4a0689..2593add406 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -60,7 +60,6 @@ import java.security.spec.RSAPrivateCrtKeySpec; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; From d93ba0e751ddade43299dc70c316af0ec6523f57 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Wed, 22 Feb 2023 16:29:24 +0200 Subject: [PATCH 12/32] Code simplify - remove method --- .../rule/engine/credentials/CertPemCredentials.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java index 2593add406..0137887aaf 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/credentials/CertPemCredentials.java @@ -133,7 +133,7 @@ public class CertPemCredentials implements ClientCredentials { KeyStore clientKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); clientKeyStore.load(null, null); for (X509Certificate certHolder : certHolders) { - clientKeyStore.setCertificateEntry(createCertEntryAlias("cert-", certHolder), certHolder); + clientKeyStore.setCertificateEntry("cert-" + certHolder.getSubjectDN().getName(), certHolder); } clientKeyStore.setKeyEntry("private-key", privateKey, @@ -150,7 +150,7 @@ public class CertPemCredentials implements ClientCredentials { KeyStore caKeyStore = KeyStore.getInstance(KeyStore.getDefaultType()); caKeyStore.load(null, null); for (X509Certificate caCertHolder : caCertHolders) { - caKeyStore.setCertificateEntry(createCertEntryAlias("caCert-cert-", caCertHolder), caCertHolder); + caKeyStore.setCertificateEntry("caCert-cert-" + caCertHolder.getSubjectDN().getName(), caCertHolder); } TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); @@ -158,10 +158,6 @@ public class CertPemCredentials implements ClientCredentials { return trustManagerFactory; } - private String createCertEntryAlias(String prefix, X509Certificate certificate) { - return prefix + "-" + certificate.getSubjectDN().getName(); - } - List readCertFile(String fileContent) throws Exception { if (fileContent == null || fileContent.trim().isEmpty()) { return Collections.emptyList(); From 680f8662ed091de5f8e62cc6eed1772276a18dfd Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Thu, 23 Feb 2023 01:01:37 +0100 Subject: [PATCH 13/32] fixed processJsonTestGatewayRequestAttributesValuesFromTheServer --- .../AbstractMqttAttributesIntegrationTest.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index 2c0278f689..6d19e1261d 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.transport.mqtt.mqttv3.attributes; +import com.fasterxml.jackson.core.type.TypeReference; import com.github.os72.protobuf.dynamic.DynamicSchema; import com.google.protobuf.Descriptors; import com.google.protobuf.DynamicMessage; @@ -22,6 +23,7 @@ import com.google.protobuf.InvalidProtocolBufferException; import com.squareup.wire.schema.internal.parser.ProtoFileElement; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.extern.slf4j.Slf4j; +import org.awaitility.Awaitility; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.DynamicProtoUtils; @@ -45,6 +47,7 @@ import org.thingsboard.server.transport.mqtt.mqttv3.MqttTestClient; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -391,9 +394,19 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt 100); assertNotNull(device); + String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; + + String attributeValuesUrl = "/api/plugins/telemetry/DEVICE/" + device.getId() + "/values/attributes/CLIENT_SCOPE?keys=" + clientKeysStr; + + Awaitility.await() + .atMost(10, TimeUnit.SECONDS) + .until(() -> { + List> attributes = doGetAsyncTyped(attributeValuesUrl, new TypeReference<>() {}); + return attributes.size() == 5; + }); + SingleEntityFilter dtf = new SingleEntityFilter(); dtf.setSingleEntity(device.getId()); - String clientKeysStr = "clientStr,clientBool,clientDbl,clientLong,clientJson"; String sharedKeysStr = "sharedStr,sharedBool,sharedDbl,sharedLong,sharedJson"; List clientKeysList = List.of(clientKeysStr.split(",")); List sharedKeysList = List.of(sharedKeysStr.split(",")); From 4fd2a41087c68f397882e454098499a431155c76 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 24 Feb 2023 11:15:39 +0200 Subject: [PATCH 14/32] Remove EventDeduplicationExecutor --- .../utils/EventDeduplicationExecutor.java | 85 --------- .../util/EventDeduplicationExecutorTest.java | 173 ------------------ 2 files changed, 258 deletions(-) delete mode 100644 application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java delete mode 100644 application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java diff --git a/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java b/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java deleted file mode 100644 index 7ce958ef2d..0000000000 --- a/application/src/main/java/org/thingsboard/server/utils/EventDeduplicationExecutor.java +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.utils; - -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.function.Consumer; - -/** - * This class deduplicate executions of the specified function. - * Useful in cluster mode, when you get event about partition change multiple times. - * Assuming that the function execution is expensive, we should execute it immediately when first time event occurs and - * later, once the processing of first event is done, process last pending task. - * - * @param

parameters of the function - */ -@Slf4j -public class EventDeduplicationExecutor

{ - private final String name; - private final ExecutorService executor; - private final Consumer

function; - private P pendingTask; - private boolean busy; - - public EventDeduplicationExecutor(String name, ExecutorService executor, Consumer

function) { - this.name = name; - this.executor = executor; - this.function = function; - } - - public void submit(P params) { - log.info("[{}] Going to submit: {}", name, params); - synchronized (EventDeduplicationExecutor.this) { - if (!busy) { - busy = true; - pendingTask = null; - try { - log.info("[{}] Submitting task: {}", name, params); - executor.submit(() -> { - try { - log.info("[{}] Executing task: {}", name, params); - function.accept(params); - } catch (Throwable e) { - log.warn("[{}] Failed to process task with parameters: {}", name, params, e); - throw e; - } finally { - unlockAndProcessIfAny(); - } - }); - } catch (Throwable e) { - log.warn("[{}] Failed to submit task with parameters: {}", name, params, e); - unlockAndProcessIfAny(); - throw e; - } - } else { - log.info("[{}] Task is already in progress. {} pending task: {}", name, pendingTask == null ? "adding" : "updating", params); - pendingTask = params; - } - } - } - - private void unlockAndProcessIfAny() { - synchronized (EventDeduplicationExecutor.this) { - busy = false; - if (pendingTask != null) { - submit(pendingTask); - } - } - } -} diff --git a/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java b/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java deleted file mode 100644 index ef75657ed4..0000000000 --- a/application/src/test/java/org/thingsboard/server/util/EventDeduplicationExecutorTest.java +++ /dev/null @@ -1,173 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.util; - -import com.google.common.util.concurrent.MoreExecutors; -import lombok.extern.slf4j.Slf4j; -import org.junit.After; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnitRunner; -import org.thingsboard.common.util.ThingsBoardThreadFactory; -import org.thingsboard.server.utils.EventDeduplicationExecutor; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.Consumer; - -@Slf4j -@RunWith(MockitoJUnitRunner.class) -public class EventDeduplicationExecutorTest { - - ThingsBoardThreadFactory threadFactory = ThingsBoardThreadFactory.forName(getClass().getSimpleName()); - ExecutorService executor; - - @After - public void tearDown() throws Exception { - if (executor != null) { - executor.shutdownNow(); - } - } - - @Test - public void testSimpleFlowSameThread() throws InterruptedException { - simpleFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testPeriodicFlowSameThread() throws InterruptedException { - periodicFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testExceptionFlowSameThread() throws InterruptedException { - exceptionFlow(MoreExecutors.newDirectExecutorService()); - } - - @Test - public void testSimpleFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - simpleFlow(executor); - } - - @Test - public void testPeriodicFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - periodicFlow(executor); - } - - @Test - public void testExceptionFlowSingleThread() throws InterruptedException { - executor = Executors.newSingleThreadExecutor(threadFactory); - exceptionFlow(executor); - } - - @Test - public void testSimpleFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - simpleFlow(executor); - } - - @Test - public void testPeriodicFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - periodicFlow(executor); - } - - @Test - public void testExceptionFlowMultiThread() throws InterruptedException { - executor = Executors.newFixedThreadPool(3, threadFactory); - exceptionFlow(executor); - } - - private void simpleFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - executor.submit(params1); - executor.submit(params2); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params1); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - private void periodicFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - executor.submit(params1); - Thread.sleep(500); - executor.submit(params2); - Thread.sleep(500); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params1); - Mockito.verify(function).accept(params2); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - private void exceptionFlow(ExecutorService executorService) throws InterruptedException { - try { - Consumer function = Mockito.spy(StringConsumer.class); - EventDeduplicationExecutor executor = new EventDeduplicationExecutor<>(EventDeduplicationExecutorTest.class.getSimpleName(), executorService, function); - - String params1 = "params1"; - String params2 = "params2"; - String params3 = "params3"; - - Mockito.doThrow(new RuntimeException()).when(function).accept("params1"); - executor.submit(params1); - executor.submit(params2); - Thread.sleep(500); - executor.submit(params3); - Thread.sleep(500); - Mockito.verify(function).accept(params2); - Mockito.verify(function).accept(params3); - } finally { - executorService.shutdownNow(); - } - } - - public static class StringConsumer implements Consumer { - @Override - public void accept(String s) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } - -} From 8a121b6b63aa3b295e348c51395429e048d1c457 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Feb 2023 20:23:26 +0100 Subject: [PATCH 15/32] tests: docker compose selenium - host.docker.internal:host-gateway --- msa/black-box-tests/src/test/resources/docker-selenium.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/resources/docker-selenium.yml b/msa/black-box-tests/src/test/resources/docker-selenium.yml index 38a3b74bfe..12b0489cd5 100644 --- a/msa/black-box-tests/src/test/resources/docker-selenium.yml +++ b/msa/black-box-tests/src/test/resources/docker-selenium.yml @@ -32,6 +32,4 @@ services: SE_SCREEN_DEPTH: 24 SE_SCREEN_DPI: 74 extra_hosts: - - "host.docker.internal:172.17.0.1" - - + - "host.docker.internal:host-gateway" From 60973c94221191d7d3786ea2ea0b03343b0a0a3b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Feb 2023 22:55:29 +0100 Subject: [PATCH 16/32] us tests: wait timeout increased up to 30 sec for slow or busy environments --- .../org/thingsboard/server/msa/ui/base/AbstractBasePage.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java index f8631bd198..cd4ca51698 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ui/base/AbstractBasePage.java @@ -32,9 +32,11 @@ import java.util.List; import java.util.Random; import java.util.UUID; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; @Slf4j abstract public class AbstractBasePage { + public static final long WAIT_TIMEOUT = TimeUnit.SECONDS.toMillis(30); protected WebDriver driver; protected WebDriverWait wait; protected Actions actions; @@ -43,7 +45,7 @@ abstract public class AbstractBasePage { public AbstractBasePage(WebDriver driver) { this.driver = driver; - this.wait = new WebDriverWait(driver, Duration.ofMillis(8000)); + this.wait = new WebDriverWait(driver, Duration.ofMillis(WAIT_TIMEOUT)); this.actions = new Actions(driver); this.js = (JavascriptExecutor) driver; } From 0f324d113297ea7ffb56df3ab59172af423868ed Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 24 Feb 2023 11:02:29 +0100 Subject: [PATCH 17/32] HashPartitionService: set myPartitions atomically --- .../server/queue/discovery/HashPartitionService.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java index 31f81d5510..3fc1626f52 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/discovery/HashPartitionService.java @@ -68,7 +68,7 @@ public class HashPartitionService implements PartitionService { private final TenantRoutingInfoService tenantRoutingInfoService; private final QueueRoutingInfoService queueRoutingInfoService; - private ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); + private volatile ConcurrentMap> myPartitions = new ConcurrentHashMap<>(); private final ConcurrentMap partitionTopicsMap = new ConcurrentHashMap<>(); private final ConcurrentMap partitionSizesMap = new ConcurrentHashMap<>(); @@ -217,17 +217,19 @@ public class HashPartitionService implements PartitionService { } queueServicesMap.values().forEach(list -> list.sort(Comparator.comparing(ServiceInfo::getServiceId))); - ConcurrentMap> oldPartitions = myPartitions; - myPartitions = new ConcurrentHashMap<>(); + final ConcurrentMap> newPartitions = new ConcurrentHashMap<>(); partitionSizesMap.forEach((queueKey, size) -> { for (int i = 0; i < size; i++) { ServiceInfo serviceInfo = resolveByPartitionIdx(queueServicesMap.get(queueKey), queueKey, i); if (currentService.equals(serviceInfo)) { - myPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); + newPartitions.computeIfAbsent(queueKey, key -> new ArrayList<>()).add(i); } } }); + final ConcurrentMap> oldPartitions = myPartitions; + myPartitions = newPartitions; + oldPartitions.forEach((queueKey, partitions) -> { if (!myPartitions.containsKey(queueKey)) { log.info("[{}] NO MORE PARTITIONS FOR CURRENT KEY", queueKey); From 5edaec55a6a91df8070184d388c84cea1cd18be4 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 25 Feb 2023 15:15:03 +0100 Subject: [PATCH 18/32] ui tests: get reliable service url inside containers network --- .../test/java/org/thingsboard/server/msa/TestProperties.java | 5 ++++- msa/black-box-tests/src/test/resources/docker-selenium.yml | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java index 32d5b0f9d7..be9a5c422a 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -16,6 +16,7 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; +import org.testcontainers.DockerClientFactory; import java.io.IOException; import java.io.InputStream; @@ -41,7 +42,9 @@ public class TestProperties { public static String getBaseUiUrl() { if (instance.isActive()) { - return "https://host.docker.internal"; + //return "https://host.docker.internal" // this alternative requires docker-selenium.yml extra_hosts: - "host.docker.internal:host-gateway" + //return "https://" + DockerClientFactory.instance().dockerHostIpAddress(); //this alternative will get Docker IP from testcontainers + return "https://haproxy"; //communicate inside current docker-compose network to the load balancer container } return getProperties().getProperty("tb.baseUiUrl"); } diff --git a/msa/black-box-tests/src/test/resources/docker-selenium.yml b/msa/black-box-tests/src/test/resources/docker-selenium.yml index 12b0489cd5..2a88264bc8 100644 --- a/msa/black-box-tests/src/test/resources/docker-selenium.yml +++ b/msa/black-box-tests/src/test/resources/docker-selenium.yml @@ -31,5 +31,6 @@ services: SE_SCREEN_HEIGHT: 1080 SE_SCREEN_DEPTH: 24 SE_SCREEN_DPI: 74 - extra_hosts: - - "host.docker.internal:host-gateway" +# Alternative way how to connect to the host address +# extra_hosts: +# - "host.docker.internal:host-gateway" From 028b1a023d3d4196ef07705b363b27bf4817b8fe Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 27 Feb 2023 14:21:39 +0100 Subject: [PATCH 19/32] fixed filtering by entity name --- .../query/DefaultEntityQueryRepository.java | 10 + .../dao/service/BaseEntityServiceTest.java | 360 +++++++++++++++++- 2 files changed, 366 insertions(+), 4 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java index 681efe4829..821dbcc011 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultEntityQueryRepository.java @@ -805,6 +805,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { private String entityNameQuery(QueryContext ctx, EntityNameFilter filter) { ctx.addStringParameter("entity_filter_name_filter", filter.getEntityNameFilter()); + + if (filter.getEntityNameFilter().startsWith("%") || filter.getEntityNameFilter().endsWith("%")) { + return "lower(e.search_text) like lower(:entity_filter_name_filter)"; + } + return "lower(e.search_text) like lower(concat(:entity_filter_name_filter, '%%'))"; } @@ -833,6 +838,11 @@ public class DefaultEntityQueryRepository implements EntityQueryRepository { } ctx.addStringParameter("entity_filter_type_query_type", type); ctx.addStringParameter("entity_filter_type_query_name", name); + + if (name.startsWith("%") || name.endsWith("%")) { + return "e.type = :entity_filter_type_query_type and lower(e.search_text) like lower(:entity_filter_type_query_name)"; + } + return "e.type = :entity_filter_type_query_type and lower(e.search_text) like lower(concat(:entity_filter_type_query_name, '%%'))"; } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java index e2003ed7b2..7c80bef095 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseEntityServiceTest.java @@ -26,7 +26,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.ResultSetExtractor; import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; @@ -50,6 +49,7 @@ import org.thingsboard.server.common.data.kv.LongDataEntry; import org.thingsboard.server.common.data.kv.StringDataEntry; import org.thingsboard.server.common.data.page.PageData; import org.thingsboard.server.common.data.query.AssetSearchQueryFilter; +import org.thingsboard.server.common.data.query.AssetTypeFilter; import org.thingsboard.server.common.data.query.DeviceSearchQueryFilter; import org.thingsboard.server.common.data.query.DeviceTypeFilter; import org.thingsboard.server.common.data.query.EdgeSearchQueryFilter; @@ -62,6 +62,7 @@ import org.thingsboard.server.common.data.query.EntityDataSortOrder; import org.thingsboard.server.common.data.query.EntityKey; import org.thingsboard.server.common.data.query.EntityKeyType; import org.thingsboard.server.common.data.query.EntityListFilter; +import org.thingsboard.server.common.data.query.EntityNameFilter; import org.thingsboard.server.common.data.query.FilterPredicateValue; import org.thingsboard.server.common.data.query.KeyFilter; import org.thingsboard.server.common.data.query.NumericFilterPredicate; @@ -106,9 +107,6 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { private TenantId tenantId; - @Autowired - private JdbcTemplate template; - @Autowired private RelationRepository relationRepository; @@ -986,6 +984,360 @@ public abstract class BaseEntityServiceTest extends AbstractServiceTest { assertEquals(devices.size(), result.getTotalElements()); } + @Test + public void testFindEntityDataByQuery_filter_entity_name_starts_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("Device"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%Device"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_entity_name_ends_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_entity_name_contains() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device test" + i); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + EntityNameFilter deviceTypeFilter = new EntityNameFilter(); + deviceTypeFilter.setEntityType(EntityType.DEVICE); + deviceTypeFilter.setEntityNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setEntityNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_starts_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("Device"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%Device%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%Device"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_ends_with() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device " + i + " test"); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_device_type_name_contains() { + List devices = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Device device = new Device(); + device.setTenantId(tenantId); + device.setName("Device test" + i); + device.setType("default"); + devices.add(device); + } + + devices.forEach(deviceService::saveDevice); + + DeviceTypeFilter deviceTypeFilter = new DeviceTypeFilter(); + deviceTypeFilter.setDeviceType("default"); + deviceTypeFilter.setDeviceNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(deviceTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(devices.size(), result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + deviceTypeFilter.setDeviceNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_starts_with() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset " + i + " test"); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("Asset"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("Asset%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%Asset%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%Asset"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_ends_with() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset " + i + " test"); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("%test"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%test%"); + + result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + + @Test + public void testFindEntityDataByQuery_filter_asset_type_name_contains() { + List assets = new ArrayList<>(); + + for (int i = 0; i < 10; i++) { + Asset asset = new Asset(); + asset.setTenantId(tenantId); + asset.setName("Asset test" + i); + asset.setType("default"); + assets.add(asset); + } + + assets.forEach(assetService::saveAsset); + + AssetTypeFilter assetTypeFilter = new AssetTypeFilter(); + assetTypeFilter.setAssetType("default"); + assetTypeFilter.setAssetNameFilter("%test%"); + + EntityDataPageLink pageLink = new EntityDataPageLink(1000, 0, null, null); + + EntityDataQuery query = new EntityDataQuery(assetTypeFilter, pageLink, null, null, null); + + PageData result = searchEntities(query); + assertEquals(assets.size(), result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("test%"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + + assetTypeFilter.setAssetNameFilter("%test"); + + result = searchEntities(query); + assertEquals(0, result.getTotalElements()); + } + private PageData searchEntities(EntityDataQuery query) { return entityService.findEntityDataByQuery(tenantId, new CustomerId(CustomerId.NULL_UUID), query); } From f823021ab8f4f075473cccb80bff5a70fd6ee93e Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 13:39:55 +0100 Subject: [PATCH 20/32] msa black box tests: cassandra limited to MAX_HEAP_SIZE: 1024M --- .../server/msa/ContainerTestSuite.java | 1 + .../server/msa/ThingsBoardDbInstaller.java | 3 +++ .../docker-compose.hybrid-test-extras.yml | 23 +++++++++++++++++++ .../docker-compose.postgres-test-extras.yml | 19 +++++++++++++++ 4 files changed, 46 insertions(+) create mode 100644 msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml create mode 100644 msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 95e2877904..ffbe14ed52 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -106,6 +106,7 @@ public class ContainerTestSuite { new File(targetDir + "docker-compose.yml"), new File(targetDir + "docker-compose.volumes.yml"), new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), + new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid-test-extras.yml" : "docker-compose.postgres-test-extras.yml")), new File(targetDir + "docker-compose.postgres.volumes.yml"), new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 4b07a72772..e6c9856094 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -82,6 +82,9 @@ public class ThingsBoardDbInstaller { )); if (IS_HYBRID_MODE) { composeFiles.add(new File("./../../docker/docker-compose.cassandra.volumes.yml")); + composeFiles.add(new File("src/test/resources/docker-compose.hybrid-test-extras.yml")); + } else { + composeFiles.add(new File("src/test/resources/docker-compose.postgres-test-extras.yml")); } String identifier = Base58.randomString(6).toLowerCase(); diff --git a/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml new file mode 100644 index 0000000000..3b5e61b47f --- /dev/null +++ b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml @@ -0,0 +1,23 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +services: + cassandra: + environment: + HEAP_NEWSIZE: 128M + MAX_HEAP_SIZE: 1024M diff --git a/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml new file mode 100644 index 0000000000..6576aa47b7 --- /dev/null +++ b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml @@ -0,0 +1,19 @@ +# +# Copyright © 2016-2023 The Thingsboard Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: '2.2' + +# Placeholder From 6fc6ef7b0627d321fc3da5750e66368e36892599 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 17:42:30 +0100 Subject: [PATCH 21/32] msa black-box-tests replaced docker compose version '2.2' -> '3.0' --- .../src/test/resources/docker-compose.hybrid-test-extras.yml | 2 +- .../src/test/resources/docker-compose.postgres-test-extras.yml | 2 +- .../src/test/resources/docker-compose.rabbitmq-server.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml index 3b5e61b47f..5df6d2582d 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.hybrid-test-extras.yml @@ -14,7 +14,7 @@ # limitations under the License. # -version: '2.2' +version: '3.0' services: cassandra: diff --git a/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml index 6576aa47b7..90d7a351cf 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.postgres-test-extras.yml @@ -14,6 +14,6 @@ # limitations under the License. # -version: '2.2' +version: '3.0' # Placeholder diff --git a/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml b/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml index 414aef5568..f1c2f0b41c 100644 --- a/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml +++ b/msa/black-box-tests/src/test/resources/docker-compose.rabbitmq-server.yml @@ -14,7 +14,7 @@ # limitations under the License. # -version: '2.2' +version: '3.0' services: rabbitmq: From 7e1ba7dec5514e0c1cbac321760d53c9fb4c5357 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 23 Feb 2023 15:55:27 +0100 Subject: [PATCH 22/32] Test mail service replaced with MockBean to fix the app lifecycle --- .../server/controller/AbstractWebTest.java | 44 ++++++++++++-- .../controller/BaseAdminControllerTest.java | 22 ++----- .../controller/BaseUserControllerTest.java | 17 +++--- .../server/service/mail/TestMailService.java | 58 ------------------- 4 files changed, 55 insertions(+), 86 deletions(-) delete mode 100644 application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 24ce820760..ab92766a27 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -36,7 +36,10 @@ import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; import org.junit.runner.Description; import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.SpyBean; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.converter.HttpMessageConverter; @@ -52,6 +55,9 @@ import org.springframework.test.web.servlet.request.MockHttpServletRequestBuilde import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.context.WebApplicationContext; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.rule.engine.api.MailService; +import org.thingsboard.rule.engine.api.util.TbNodeUtils; import org.thingsboard.server.common.data.Customer; import org.thingsboard.server.common.data.DeviceProfile; import org.thingsboard.server.common.data.DeviceProfileType; @@ -69,6 +75,7 @@ import org.thingsboard.server.common.data.device.profile.MqttTopics; import org.thingsboard.server.common.data.device.profile.ProtoTransportPayloadConfiguration; import org.thingsboard.server.common.data.device.profile.TransportPayloadTypeConfiguration; import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.exception.ThingsboardException; import org.thingsboard.server.common.data.id.CustomerId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.HasId; @@ -83,7 +90,6 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.config.ThingsboardSecurityConfiguration; import org.thingsboard.server.dao.Dao; import org.thingsboard.server.dao.tenant.TenantProfileService; -import org.thingsboard.server.service.mail.TestMailService; import org.thingsboard.server.service.security.auth.jwt.RefreshTokenRequest; import org.thingsboard.server.service.security.auth.rest.LoginRequest; @@ -98,6 +104,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.asyncDispatch; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; @@ -144,6 +151,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { protected MockMvc mockMvc; + protected String currentActivateToken; + protected String currentResetPasswordToken; + protected String token; protected String refreshToken; protected String username; @@ -168,6 +178,9 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { @Autowired private TenantProfileService tenantProfileService; + @SpyBean + protected MailService mailService; + @Rule public TestRule watcher = new TestWatcher() { protected void starting(Description description) { @@ -200,6 +213,8 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { public void setupWebTest() throws Exception { log.info("Executing web test setup"); + setupMailServiceMock(); + if (this.mockMvc == null) { this.mockMvc = webAppContextSetup(webApplicationContext) .apply(springSecurity()).build(); @@ -241,6 +256,27 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { log.info("Executed web test setup"); } + private void setupMailServiceMock() throws ThingsboardException { + Mockito.doNothing().when(mailService).sendAccountActivatedEmail(anyString(), anyString()); + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + String activationLink = (String) args[0]; + currentActivateToken = activationLink.split("=")[1]; + return null; + } + }).when(mailService).sendActivationEmail(anyString(), anyString()); + + Mockito.doAnswer(new Answer() { + public Void answer(InvocationOnMock invocation) { + Object[] args = invocation.getArguments(); + String passwordResetLink = (String) args[0]; + currentResetPasswordToken = passwordResetLink.split("=")[1]; + return null; + } + }).when(mailService).sendResetPasswordEmailAsync(anyString(), anyString()); + } + @After public void teardownWebTest() throws Exception { log.info("Executing web test teardown"); @@ -368,11 +404,11 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { } private JsonNode getActivateRequest(String password) throws Exception { - doGet("/api/noauth/activate?activateToken={activateToken}", TestMailService.currentActivateToken) + doGet("/api/noauth/activate?activateToken={activateToken}", this.currentActivateToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + TestMailService.currentActivateToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + this.currentActivateToken)); return new ObjectMapper().createObjectNode() - .put("activateToken", TestMailService.currentActivateToken) + .put("activateToken", this.currentActivateToken) .put("password", password); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java index 021171f1af..e6c27aa4a8 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java @@ -21,12 +21,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Test; import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.security.model.JwtSettings; -import org.thingsboard.server.service.mail.DefaultMailService; import java.nio.charset.StandardCharsets; import java.util.Base64; @@ -35,6 +32,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -43,12 +42,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. public abstract class BaseAdminControllerTest extends AbstractControllerTest { final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "thingsboardDefaultSigningKey"); - @Autowired - MailService mailService; - - @Autowired - DefaultMailService defaultMailService; - @Test public void testFindAdminSettingsByKey() throws Exception { loginSysAdmin(); @@ -118,10 +111,12 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { @Test public void testSendTestMail() throws Exception { + Mockito.doNothing().when(mailService).sendTestMail(any(), anyString()); loginSysAdmin(); AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); doPost("/api/admin/settings/testMail", adminSettings) .andExpect(status().isOk()); + Mockito.verify(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); } @Test @@ -137,15 +132,8 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { adminSettings.setJsonValue(objectNode); - Mockito.doAnswer((invocations) -> { - var jsonConfig = (JsonNode) invocations.getArgument(0); - var email = (String) invocations.getArgument(1); - - defaultMailService.sendTestMail(jsonConfig, email); - return null; - }).when(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); doPost("/api/admin/settings/testMail", adminSettings).andExpect(status().is5xxServerError()); - Mockito.doNothing().when(mailService).sendTestMail(Mockito.any(), Mockito.any()); + Mockito.verify(mailService).sendTestMail(Mockito.any(), Mockito.anyString()); } void resetJwtSettingsToDefault() throws Exception { diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java index f289c8066f..7149f52630 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseUserControllerTest.java @@ -44,7 +44,6 @@ import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.dao.exception.DataValidationException; import org.thingsboard.server.dao.user.UserDao; -import org.thingsboard.server.service.mail.TestMailService; import java.util.ArrayList; import java.util.Collections; @@ -54,6 +53,8 @@ import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.header; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; @@ -106,12 +107,12 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { Mockito.reset(tbClusterService, auditLogService); resetTokens(); - doGet("/api/noauth/activate?activateToken={activateToken}", TestMailService.currentActivateToken) + doGet("/api/noauth/activate?activateToken={activateToken}", this.currentActivateToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + TestMailService.currentActivateToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/createPassword?activateToken=" + this.currentActivateToken)); JsonNode activateRequest = new ObjectMapper().createObjectNode() - .put("activateToken", TestMailService.currentActivateToken) + .put("activateToken", this.currentActivateToken) .put("password", "testPassword"); JsonNode tokenInfo = readResponse(doPost("/api/noauth/activate", activateRequest).andExpect(status().isOk()), JsonNode.class); @@ -208,17 +209,19 @@ public abstract class BaseUserControllerTest extends AbstractControllerTest { doPost("/api/noauth/resetPasswordByEmail", resetPasswordByEmailRequest) .andExpect(status().isOk()); Thread.sleep(1000); - doGet("/api/noauth/resetPassword?resetToken={resetToken}", TestMailService.currentResetPasswordToken) + doGet("/api/noauth/resetPassword?resetToken={resetToken}", this.currentResetPasswordToken) .andExpect(status().isSeeOther()) - .andExpect(header().string(HttpHeaders.LOCATION, "/login/resetPassword?resetToken=" + TestMailService.currentResetPasswordToken)); + .andExpect(header().string(HttpHeaders.LOCATION, "/login/resetPassword?resetToken=" + this.currentResetPasswordToken)); JsonNode resetPasswordRequest = new ObjectMapper().createObjectNode() - .put("resetToken", TestMailService.currentResetPasswordToken) + .put("resetToken", this.currentResetPasswordToken) .put("password", "testPassword2"); + Mockito.doNothing().when(mailService).sendPasswordWasResetEmail(anyString(), anyString()); JsonNode tokenInfo = readResponse( doPost("/api/noauth/resetPassword", resetPasswordRequest) .andExpect(status().isOk()), JsonNode.class); + Mockito.verify(mailService).sendPasswordWasResetEmail(anyString(), anyString()); validateAndSetJwtToken(tokenInfo, email); doGet("/api/auth/user") diff --git a/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java b/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java deleted file mode 100644 index 7c4c45c951..0000000000 --- a/application/src/test/java/org/thingsboard/server/service/mail/TestMailService.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Copyright © 2016-2023 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.mail; - -import org.mockito.Mockito; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.context.annotation.Profile; -import org.thingsboard.rule.engine.api.MailService; -import org.thingsboard.server.common.data.exception.ThingsboardException; - -@Profile("test") -@Configuration -public class TestMailService { - - public static String currentActivateToken; - public static String currentResetPasswordToken; - - @Bean - @Primary - public MailService mailService() throws ThingsboardException { - MailService mailService = Mockito.mock(MailService.class); - Mockito.doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String activationLink = (String) args[0]; - currentActivateToken = activationLink.split("=")[1]; - return null; - } - }).when(mailService).sendActivationEmail(Mockito.anyString(), Mockito.anyString()); - Mockito.doAnswer(new Answer() { - public Void answer(InvocationOnMock invocation) { - Object[] args = invocation.getArguments(); - String passwordResetLink = (String) args[0]; - currentResetPasswordToken = passwordResetLink.split("=")[1]; - return null; - } - }).when(mailService).sendResetPasswordEmailAsync(Mockito.anyString(), Mockito.anyString()); - return mailService; - } - -} From a6a4a67a3d3b644803ed23ecaf9c6b6d9b7d2cb0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 17 Feb 2023 17:50:50 +0100 Subject: [PATCH 23/32] handle timeouts on transport tests to provide clear explanation what is the problem. timeout increased for run test stable on slow (busy) environments --- .../AbstractTransportIntegrationTest.java | 2 +- ...AbstractMqttAttributesIntegrationTest.java | 49 ++++++++++++------- 2 files changed, 32 insertions(+), 19 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java index cf4de21cc2..89a4b707bf 100644 --- a/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/AbstractTransportIntegrationTest.java @@ -28,7 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; @Slf4j public abstract class AbstractTransportIntegrationTest extends AbstractControllerTest { - protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 10; + protected static final int DEFAULT_WAIT_TIMEOUT_SECONDS = 30; protected static final String MQTT_URL = "tcp://localhost:1883"; protected static final String COAP_BASE_URL = "coap://localhost:5683/api/v1/"; diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java index 6d19e1261d..7feafead95 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/mqttv3/attributes/AbstractMqttAttributesIntegrationTest.java @@ -128,14 +128,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateUpdateAttributesJsonResponse(onUpdateCallback, SHARED_ATTRIBUTES_PAYLOAD); MqttTestCallback onDeleteCallback = new MqttTestCallback(); client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateUpdateAttributesJsonResponse(onDeleteCallback, SHARED_ATTRIBUTES_DELETED_RESPONSE); client.disconnect(); } @@ -148,13 +150,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(attrSubTopic, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateUpdateAttributesProtoResponse(onUpdateCallback); MqttTestCallback onDeleteCallback = new MqttTestCallback(); client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateDeleteAttributesProtoResponse(onDeleteCallback); client.disconnect(); } @@ -165,7 +169,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateUpdateAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); List tsKvProtoList = getTsKvProtoList("shared"); attributeUpdateNotificationMsgBuilder.addAllSharedUpdated(tsKvProtoList); @@ -181,7 +185,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateDeleteAttributesProtoResponse(MqttTestCallback callback) throws InvalidProtocolBufferException { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); @@ -212,7 +216,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.subscribeAndWait(GATEWAY_ATTRIBUTES_TOPIC, MqttQoS.AT_MOST_ONCE); doPostAsync("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/attributes/SHARED_SCOPE", SHARED_ATTRIBUTES_PAYLOAD, String.class, status().isOk()); - onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onUpdateCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onUpdateCallback").isTrue(); validateJsonGatewayUpdateAttributesResponse(onUpdateCallback, deviceName, SHARED_ATTRIBUTES_PAYLOAD); @@ -220,7 +225,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt client.setCallback(onDeleteCallback); doDelete("/api/plugins/telemetry/DEVICE/" + savedDevice.getId().getId() + "/SHARED_SCOPE?keys=sharedJson", String.class); - onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(onDeleteCallback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await onDeleteCallback").isTrue(); validateJsonGatewayUpdateAttributesResponse(onDeleteCallback, deviceName, SHARED_ATTRIBUTES_DELETED_RESPONSE); client.disconnect(); @@ -249,7 +255,7 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName, String expectResultData) { - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); assertEquals(JacksonUtil.toJsonNode(getGatewayAttributesResponseJson(deviceName, expectResultData)), JacksonUtil.fromBytes(callback.getPayloadBytes())); } @@ -263,8 +269,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoGatewayUpdateAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); List tsKvProtoList = getTsKvProtoList("shared"); @@ -288,8 +295,9 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoGatewayDeleteAttributesResponse(MqttTestCallback callback, String deviceName) throws InvalidProtocolBufferException, InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); - assertNotNull(callback.getPayloadBytes()); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); + assertThat(callback.getPayloadBytes()).as("callback payload non-null").isNotNull(); TransportProtos.AttributeUpdateNotificationMsg.Builder attributeUpdateNotificationMsgBuilder = TransportProtos.AttributeUpdateNotificationMsg.newBuilder(); attributeUpdateNotificationMsgBuilder.addSharedDeleted("sharedJson"); TransportProtos.AttributeUpdateNotificationMsg attributeUpdateNotificationMsg = attributeUpdateNotificationMsgBuilder.build(); @@ -551,13 +559,15 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonResponse(MqttTestCallback callback, String expectedResponse) throws InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); assertEquals(JacksonUtil.toJsonNode(expectedResponse), JacksonUtil.fromBytes(callback.getPayloadBytes())); } protected void validateProtoResponse(MqttTestCallback callback, TransportProtos.GetAttributeResponseMsg expectedResponse) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_MOST_ONCE.value(), callback.getQoS()); TransportProtos.GetAttributeResponseMsg actualAttributesResponse = TransportProtos.GetAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); assertEquals(expectedResponse.getRequestId(), actualAttributesResponse.getRequestId()); @@ -580,14 +590,16 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateJsonResponseGateway(MqttTestCallback callback, String deviceName, String expectedValues) throws InterruptedException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); String expectedRequestPayload = "{\"id\":1,\"device\":\"" + deviceName + "\",\"values\":" + expectedValues + "}"; assertEquals(JacksonUtil.toJsonNode(expectedRequestPayload), JacksonUtil.fromBytes(callback.getPayloadBytes())); } protected void validateProtoClientResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, true); TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); @@ -603,7 +615,8 @@ public abstract class AbstractMqttAttributesIntegrationTest extends AbstractMqtt } protected void validateProtoSharedResponseGateway(MqttTestCallback callback, String deviceName) throws InterruptedException, InvalidProtocolBufferException { - callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + assertThat(callback.getSubscribeLatch().await(DEFAULT_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS)) + .as("await callback").isTrue(); assertEquals(MqttQoS.AT_LEAST_ONCE.value(), callback.getQoS()); TransportApiProtos.GatewayAttributeResponseMsg expectedGatewayAttributeResponseMsg = getExpectedGatewayAttributeResponseMsg(deviceName, false); TransportApiProtos.GatewayAttributeResponseMsg actualGatewayAttributeResponseMsg = TransportApiProtos.GatewayAttributeResponseMsg.parseFrom(callback.getPayloadBytes()); From c38dd1270381816f69a45b583a94c94ee2ac1bc4 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:39:08 +0100 Subject: [PATCH 24/32] test dependency upgrade: surefire.version 3.0.0-M9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 61053bebc7..ae5dba306e 100755 --- a/pom.xml +++ b/pom.xml @@ -85,7 +85,7 @@ 2.0.51.Final 1.7.0 4.8.0 - 3.0.0-M6 + 3.0.0-M9 3.0.2 3.0.4 1.6.3 From af62bc1d98361abda60f86e6688e0e3ed380f563 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:38:02 +0100 Subject: [PATCH 25/32] Shutdown jsExecutor after Nashorn destroy --- .../org/thingsboard/script/api/js/NashornJsInvokeService.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java index c995474452..a93bb018d8 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java @@ -127,6 +127,9 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { if (monitorExecutorService != null) { monitorExecutorService.shutdownNow(); } + if (jsExecutor != null) { + jsExecutor.shutdownNow(); + } } @Override From bf5f11f7c7d1e70b5470e41f8f4d43fcdea95959 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 21:36:24 +0100 Subject: [PATCH 26/32] tests: surefire plugin: -Xss384k -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=20 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae5dba306e..7d6b59f10b 100755 --- a/pom.xml +++ b/pom.xml @@ -659,7 +659,7 @@ ${surefire.version} - --illegal-access=permit + --illegal-access=permit -Xss384k -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=20 From 0448c9aa03135ddd92720b31c272eef7f0f847d7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Sat, 18 Feb 2023 00:16:12 +0100 Subject: [PATCH 27/32] test memory optimization (disable integration rpc, coap, hikari.maximumPoolSize=16, etc). --- .../server/controller/AbstractWebTest.java | 19 ++++++++++++++++--- .../BaseWidgetsBundleControllerTest.java | 2 +- .../resources/application-test.properties | 8 ++++++++ .../src/test/resources/logback-test.xml | 1 + dao/src/test/resources/nosql-test.properties | 2 +- dao/src/test/resources/sql-test.properties | 2 +- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java index 24ce820760..8e0804e3df 100644 --- a/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/AbstractWebTest.java @@ -29,8 +29,10 @@ import lombok.extern.slf4j.Slf4j; import org.hamcrest.Matcher; import org.hibernate.exception.ConstraintViolationException; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; +import org.junit.BeforeClass; import org.junit.Rule; import org.junit.rules.TestRule; import org.junit.rules.TestWatcher; @@ -196,9 +198,20 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { this.mappingJackson2HttpMessageConverter); } + @BeforeClass + public static void beforeWebTestClass() throws Exception { + + } + + @AfterClass + public static void afterWebTestClass() throws Exception { + Mockito.clearAllCaches(); + } + + @Before public void setupWebTest() throws Exception { - log.info("Executing web test setup"); + log.debug("Executing web test setup"); if (this.mockMvc == null) { this.mockMvc = webAppContextSetup(webApplicationContext) @@ -238,12 +251,12 @@ public abstract class AbstractWebTest extends AbstractInMemoryStorageTest { resetTokens(); - log.info("Executed web test setup"); + log.debug("Executed web test setup"); } @After public void teardownWebTest() throws Exception { - log.info("Executing web test teardown"); + log.debug("Executing web test teardown"); loginSysAdmin(); doDelete("/api/tenant/" + tenantId.getId().toString()) diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java index 56df8513fe..828d279e93 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseWidgetsBundleControllerTest.java @@ -192,7 +192,7 @@ public abstract class BaseWidgetsBundleControllerTest extends AbstractController WidgetsBundle savedWidgetsBundle = doPost("/api/widgetsBundle", widgetsBundle, WidgetsBundle.class); savedWidgetsBundle.setAlias("new_alias"); - Mockito.reset(tbClusterService); + Mockito.clearInvocations(tbClusterService); doPost("/api/widgetsBundle", savedWidgetsBundle) .andExpect(status().isBadRequest()) diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index e4a36c151d..330ae69787 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -20,6 +20,8 @@ transport.mqtt.enabled=false transport.coap.enabled=false transport.lwm2m.enabled=false transport.snmp.enabled=false +coap.enabled=false +integrations.rpc.enabled=false # Low latency settings to perform tests as fast as possible sql.attributes.batch_max_delay=5 @@ -63,3 +65,9 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 +#spring.jpa.properties.hibernate.generate_statistics=true +#spring.jpa.properties.hibernate.jmx.enabled=true +#spring.jpa.properties.hibernate.jmx.usePlatformServer=true +#spring.jpa.properties.hibernate.cache.use_query_cache=false +#spring.jpa.properties.hibernate.query.plan_cache_max_size=64 +#spring.jpa.properties.hibernate.query.plan_parameter_metadata_max_size=32 diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index 3762c8aa7c..c79a7d3905 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -15,6 +15,7 @@ + diff --git a/dao/src/test/resources/nosql-test.properties b/dao/src/test/resources/nosql-test.properties index 1c6d2442cf..d63a9f7f82 100644 --- a/dao/src/test/resources/nosql-test.properties +++ b/dao/src/test/resources/nosql-test.properties @@ -15,7 +15,7 @@ spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver -spring.datasource.hikari.maximumPoolSize = 50 +spring.datasource.hikari.maximumPoolSize=16 queue.rule-engine.queues[0].name=Main queue.rule-engine.queues[0].topic=tb_rule_engine.main diff --git a/dao/src/test/resources/sql-test.properties b/dao/src/test/resources/sql-test.properties index b2ded716c4..2fed33b43a 100644 --- a/dao/src/test/resources/sql-test.properties +++ b/dao/src/test/resources/sql-test.properties @@ -16,7 +16,7 @@ spring.datasource.username=postgres spring.datasource.password=postgres spring.datasource.url=jdbc:tc:postgresql:12.8:///thingsboard?TC_DAEMON=true&TC_TMPFS=/testtmpfs:rw&?TC_INITFUNCTION=org.thingsboard.server.dao.PostgreSqlInitializer::initDb spring.datasource.driverClassName=org.testcontainers.jdbc.ContainerDatabaseDriver -spring.datasource.hikari.maximumPoolSize = 50 +spring.datasource.hikari.maximumPoolSize=16 service.type=monolith From 368455119b03e7c5a829a95eed30cd83ed01bef2 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 21:58:12 +0100 Subject: [PATCH 28/32] Shutdown callback executor for RemoteJsInvokeService. Call shutdown timeoutExecutorService from AbstractScriptInvokeService --- .../server/service/script/RemoteJsInvokeService.java | 5 ++++- .../thingsboard/script/api/js/NashornJsInvokeService.java | 2 ++ .../script/api/tbel/DefaultTbelInvokeService.java | 5 ++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java b/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java index 82c806ad4f..8fff7fc0fe 100644 --- a/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java +++ b/common/script/remote-js-client/src/main/java/org/thingsboard/server/service/script/RemoteJsInvokeService.java @@ -109,17 +109,20 @@ public class RemoteJsInvokeService extends AbstractJsInvokeService { private final Lock scriptsLock = new ReentrantLock(); @PostConstruct + @Override public void init() { super.init(); requestTemplate.init(); } @PreDestroy - public void destroy() { + @Override + public void stop() { super.stop(); if (requestTemplate != null) { requestTemplate.stop(); } + callbackExecutor.shutdownNow(); } @Override diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java index c995474452..b1f1c0d2b1 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/js/NashornJsInvokeService.java @@ -104,6 +104,7 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { } @PostConstruct + @Override public void init() { super.init(); jsExecutor = MoreExecutors.listeningDecorator(Executors.newWorkStealingPool(jsExecutorThreadPoolSize)); @@ -122,6 +123,7 @@ public class NashornJsInvokeService extends AbstractJsInvokeService { } @PreDestroy + @Override public void stop() { super.stop(); if (monitorExecutorService != null) { diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java index 7d2043557e..f98e33b615 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/tbel/DefaultTbelInvokeService.java @@ -119,6 +119,7 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem @SneakyThrows @PostConstruct + @Override public void init() { super.init(); OptimizerFactory.setDefaultOptimizer(OptimizerFactory.SAFE_REFLECTIVE); @@ -142,7 +143,9 @@ public class DefaultTbelInvokeService extends AbstractScriptInvokeService implem } @PreDestroy - public void destroy() { + @Override + public void stop() { + super.stop(); if (executor != null) { executor.shutdownNow(); } From 12c0e23ed3514564e0fe547bc506574eab2bbc14 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:48:42 +0100 Subject: [PATCH 29/32] disabled service.integrations and js.evaluator for the most of cases --- .../server/service/script/NashornJsInvokeServiceTest.java | 1 + .../transport/coap/AbstractCoapIntegrationTest.java | 2 ++ .../transport/mqtt/AbstractMqttIntegrationTest.java | 2 +- .../src/test/resources/application-test.properties | 8 ++------ 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java index 08cd031a52..4373a4c108 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/NashornJsInvokeServiceTest.java @@ -36,6 +36,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest @TestPropertySource(properties = { + "js.evaluator=local", "js.max_script_body_size=50", "js.max_total_args_size=50", "js.max_result_size=50", diff --git a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java index d90af7b6ba..1266ef2c29 100644 --- a/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/coap/AbstractCoapIntegrationTest.java @@ -45,6 +45,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @TestPropertySource(properties = { + "coap.enabled=true", + "service.integrations.supported=ALL", "transport.coap.enabled=true", }) @Slf4j diff --git a/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java b/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java index 0f1815b46b..6cb6e49e79 100644 --- a/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java +++ b/application/src/test/java/org/thingsboard/server/transport/mqtt/AbstractMqttIntegrationTest.java @@ -46,8 +46,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @TestPropertySource(properties = { + "service.integrations.supported=ALL", "transport.mqtt.enabled=true", - "js.evaluator=mock", }) @Slf4j public abstract class AbstractMqttIntegrationTest extends AbstractTransportIntegrationTest { diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index 330ae69787..ad86ff736b 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -1,3 +1,4 @@ +js.evaluator=mock transport.lwm2m.server.security.credentials.enabled=true transport.lwm2m.server.security.credentials.type=KEYSTORE transport.lwm2m.server.security.credentials.keystore.store_file=lwm2m/credentials/lwm2mserver.jks @@ -22,6 +23,7 @@ transport.lwm2m.enabled=false transport.snmp.enabled=false coap.enabled=false integrations.rpc.enabled=false +service.integrations.supported=NONE # Low latency settings to perform tests as fast as possible sql.attributes.batch_max_delay=5 @@ -65,9 +67,3 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 -#spring.jpa.properties.hibernate.generate_statistics=true -#spring.jpa.properties.hibernate.jmx.enabled=true -#spring.jpa.properties.hibernate.jmx.usePlatformServer=true -#spring.jpa.properties.hibernate.cache.use_query_cache=false -#spring.jpa.properties.hibernate.query.plan_cache_max_size=64 -#spring.jpa.properties.hibernate.query.plan_parameter_metadata_max_size=32 From 24c930c699f65518f76947f4e2a36326a762cd3d Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 20 Feb 2023 17:49:08 +0100 Subject: [PATCH 30/32] spring.test.context.cache.maxSize=1 --- application/src/test/resources/application-test.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/application/src/test/resources/application-test.properties b/application/src/test/resources/application-test.properties index ad86ff736b..a4bedd9527 100644 --- a/application/src/test/resources/application-test.properties +++ b/application/src/test/resources/application-test.properties @@ -67,3 +67,4 @@ sql.ttl.audit_logs.ttl=2592000 sql.edge_events.partition_size=168 sql.ttl.edge_events.edge_event_ttl=2592000 +spring.test.context.cache.maxSize=1 \ No newline at end of file From 945d1955e2de09d61d1c4684d94b590c503268f0 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 27 Feb 2023 00:26:06 +0100 Subject: [PATCH 31/32] msa modules reordered to tolerate parallel build with 3 parallel threads --- msa/pom.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/msa/pom.xml b/msa/pom.xml index 515557b018..ad40e574ec 100644 --- a/msa/pom.xml +++ b/msa/pom.xml @@ -39,13 +39,14 @@ + tb + web-ui vc-executor vc-executor-docker - js-executor - web-ui tb-node transport + js-executor From 04c36916edb1de7666851474a79d764b7ec76b04 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Tue, 28 Feb 2023 14:53:57 +0200 Subject: [PATCH 32/32] moved system env property --- .../org/thingsboard/server/controller/AuthController.java | 2 +- application/src/main/resources/thingsboard.yml | 5 +++-- .../java/org/thingsboard/server/common/data/StringUtils.java | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index d0e382f888..113c292380 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -75,7 +75,7 @@ import java.util.concurrent.ConcurrentMap; @RequiredArgsConstructor public class AuthController extends BaseController { - @Value("${rate_limits.reset_password_per_user:5:3600}") + @Value("${server.rest.rate_limits.reset_password_per_user:5:3600}") private String defaultLimitsConfiguration; private final ConcurrentMap resetPasswordRateLimits = new ConcurrentHashMap<>(); private final BCryptPasswordEncoder passwordEncoder; diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 7a5a1e57fd..d5d3a9dbd0 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -73,6 +73,8 @@ server: min_timeout: "${MIN_SERVER_SIDE_RPC_TIMEOUT:5000}" # Default value of the server side RPC timeout. default_timeout: "${DEFAULT_SERVER_SIDE_RPC_TIMEOUT:10000}" + rate_limits: + reset_password_per_user: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:3600}" # Application info app: @@ -1209,5 +1211,4 @@ management: exposure: # Expose metrics endpoint (use value 'prometheus' to enable prometheus metrics). include: '${METRICS_ENDPOINTS_EXPOSE:info}' -rate_limits: - reset_password_per_user: "${RESET_PASSWORD_PER_USER_RATE_LIMIT_CONFIGURATION:5:3600}" + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java index 7a4c180b9c..3b38aa57c1 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/StringUtils.java @@ -24,6 +24,8 @@ import java.util.Base64; import static org.apache.commons.lang3.StringUtils.repeat; public class StringUtils { + public static final SecureRandom RANDOM = new SecureRandom(); + public static final String EMPTY = ""; public static final int INDEX_NOT_FOUND = -1; @@ -184,9 +186,8 @@ public class StringUtils { } public static String generateSafeToken(int length) { - SecureRandom random = new SecureRandom(); byte[] bytes = new byte[length]; - random.nextBytes(bytes); + RANDOM.nextBytes(bytes); Base64.Encoder encoder = Base64.getUrlEncoder().withoutPadding(); return encoder.encodeToString(bytes); }