From dc07e9ae459d6cb7f0ec99ba1664df0180d14463 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Fri, 16 Sep 2022 18:57:50 +0300 Subject: [PATCH 01/80] jwt settings --- .../server/config/JwtSettings.java | 44 +++++++++++++++++++ .../src/main/resources/thingsboard.yml | 2 +- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java index 95e510612a..1fc0b4bd8c 100644 --- a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java +++ b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java @@ -15,15 +15,30 @@ */ package org.thingsboard.server.config; +import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.JwtToken; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import javax.annotation.PostConstruct; +import java.nio.charset.StandardCharsets; +import java.util.Base64; @Component @ConfigurationProperties(prefix = "security.jwt") @Data +@Slf4j public class JwtSettings { + static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; + static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; /** * {@link JwtToken} will expire after this time. */ @@ -36,6 +51,7 @@ public class JwtSettings { /** * Key is used to sign {@link JwtToken}. + * Base64 encoded */ private String tokenSigningKey; @@ -44,4 +60,32 @@ public class JwtSettings { */ private Integer refreshTokenExpTime; + @JsonIgnore + @Autowired + private AdminSettingsService adminSettingsService; + + @PostConstruct + public void init() { + AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + if (adminJwtSettings == null) { + if (TOKEN_SIGNING_KEY_DEFAULT.equals(tokenSigningKey)) { + log.warn("JWT token signing key is default. Generating a new random key"); + tokenSigningKey = Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)); + } + adminJwtSettings = new AdminSettings(); + adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); + adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); + adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(this)); + log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); + } else { + log.debug("Loading the JWT admin settings"); + JwtSettings jwtSettings = JacksonUtil.treeToValue(adminJwtSettings.getJsonValue(), JwtSettings.class); + this.setRefreshTokenExpTime(jwtSettings.getRefreshTokenExpTime()); + this.setTokenExpirationTime(jwtSettings.getTokenExpirationTime()); + this.setTokenIssuer(jwtSettings.getTokenIssuer()); + this.setTokenSigningKey(jwtSettings.getTokenSigningKey()); + } + } + } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 17b996934e..a6e8637247 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -111,7 +111,7 @@ security: tokenExpirationTime: "${JWT_TOKEN_EXPIRATION_TIME:9000}" # Number of seconds (2.5 hours) refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:604800}" # Number of seconds (1 week) tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" - tokenSigningKey: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}" + tokenSigningKey: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}" # Base64 encoded # Enable/disable access to Tenant Administrators JWT token by System Administrator or Customer Users JWT token by Tenant Administrator user_token_access_enabled: "${SECURITY_USER_TOKEN_ACCESS_ENABLED:true}" # Enable/disable case-sensitive username login From 9b519d33a19532f4cca8973e521d5ee1186d5b35 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 19 Sep 2022 12:43:43 +0300 Subject: [PATCH 02/80] jwt settings install and upgrade --- .../server/config/JwtSettings.java | 28 ++++----- .../install/ThingsboardInstallService.java | 12 ++++ .../ConditionValidatorUpgradeService.java | 22 +++++++ .../ConditionValidatorUpgradeServiceImpl.java | 63 +++++++++++++++++++ .../DefaultSystemDataLoaderService.java | 33 ++++++++++ .../install/SystemDataLoaderService.java | 2 + .../update/DefaultDataUpdateService.java | 4 ++ .../src/main/resources/thingsboard.yml | 2 +- 8 files changed, 148 insertions(+), 18 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java create mode 100644 application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java index 1fc0b4bd8c..14e228bde8 100644 --- a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java +++ b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java @@ -18,7 +18,6 @@ package org.thingsboard.server.config; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; @@ -29,15 +28,13 @@ import org.thingsboard.server.common.data.security.model.JwtToken; import org.thingsboard.server.dao.settings.AdminSettingsService; import javax.annotation.PostConstruct; -import java.nio.charset.StandardCharsets; -import java.util.Base64; @Component @ConfigurationProperties(prefix = "security.jwt") @Data @Slf4j public class JwtSettings { - static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; + public static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; /** * {@link JwtToken} will expire after this time. @@ -67,25 +64,22 @@ public class JwtSettings { @PostConstruct public void init() { AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); - if (adminJwtSettings == null) { - if (TOKEN_SIGNING_KEY_DEFAULT.equals(tokenSigningKey)) { - log.warn("JWT token signing key is default. Generating a new random key"); - tokenSigningKey = Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8)); - } - adminJwtSettings = new AdminSettings(); - adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); - adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); - adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(this)); - log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); - } else { - log.debug("Loading the JWT admin settings"); + if (adminJwtSettings != null) { + log.debug("Loading the JWT admin settings from database"); JwtSettings jwtSettings = JacksonUtil.treeToValue(adminJwtSettings.getJsonValue(), JwtSettings.class); this.setRefreshTokenExpTime(jwtSettings.getRefreshTokenExpTime()); this.setTokenExpirationTime(jwtSettings.getTokenExpirationTime()); this.setTokenIssuer(jwtSettings.getTokenIssuer()); this.setTokenSigningKey(jwtSettings.getTokenSigningKey()); } + + if (hasDefaultTokenSigningKey()) { + log.warn("JWT token signing key is default. This is a security issue. Please, consider to set unique value"); + } + } + + public boolean hasDefaultTokenSigningKey() { + return TOKEN_SIGNING_KEY_DEFAULT.equals(tokenSigningKey); } } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 40e15c7b0e..c83552e777 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.service.install.migrate.EntitiesMigrateService; import org.thingsboard.server.service.install.migrate.TsLatestMigrateService; import org.thingsboard.server.service.install.update.CacheCleanupService; import org.thingsboard.server.service.install.update.DataUpdateService; +import org.thingsboard.server.service.install.ConditionValidatorUpgradeService; @Service @Profile("install") @@ -84,11 +85,17 @@ public class ThingsboardInstallService { @Autowired(required = false) private TsLatestMigrateService latestMigrateService; + @Autowired + private ConditionValidatorUpgradeService conditionValidatorUpgradeService; + + public void performInstall() { try { if (isUpgrade) { log.info("Starting ThingsBoard Upgrade from version {} ...", upgradeFromVersion); + conditionValidatorUpgradeService.validateConditionsBeforeUpgrade(upgradeFromVersion); + cacheCleanupService.clearCache(upgradeFromVersion); if ("2.5.0-cassandra".equals(upgradeFromVersion)) { @@ -224,6 +231,10 @@ public class ThingsboardInstallService { log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; + case "3.4.0": + log.info("Upgrading ThingsBoard from version 3.4.0 to 3.5.0 ..."); + dataUpdateService.updateData("3.4.0"); + break; //TODO update CacheCleanupService on the next version upgrade @@ -257,6 +268,7 @@ public class ThingsboardInstallService { systemDataLoaderService.createSysAdmin(); systemDataLoaderService.createDefaultTenantProfiles(); systemDataLoaderService.createAdminSettings(); + systemDataLoaderService.createJwtAdminSettings(); systemDataLoaderService.loadSystemWidgets(); systemDataLoaderService.createOAuth2Templates(); systemDataLoaderService.createQueues(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java new file mode 100644 index 0000000000..ec98a2f37e --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java @@ -0,0 +1,22 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +public interface ConditionValidatorUpgradeService { + + void validateConditionsBeforeUpgrade(String fromVersion) throws Exception; + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java new file mode 100644 index 0000000000..f70086abee --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java @@ -0,0 +1,63 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.install; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import javax.validation.ValidationException; + +import static org.thingsboard.server.config.JwtSettings.ADMIN_SETTINGS_JWT_KEY; + +@Service +@Profile("install") +@RequiredArgsConstructor +@Slf4j +public class ConditionValidatorUpgradeServiceImpl implements ConditionValidatorUpgradeService { + + private final AdminSettingsService adminSettingsService; + + private final JwtSettings jwtSettings; + + @Override + public void validateConditionsBeforeUpgrade(String fromVersion) throws Exception { + log.info("Validating conditions before upgrade.."); + validateJwtTokenSigningKey(); + } + + void validateJwtTokenSigningKey() { + AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + if (adminJwtSettings == null) { + if (jwtSettings.hasDefaultTokenSigningKey()) { + String allowDefaultJwtSigningKey = System.getenv("TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"); + if ("true".equalsIgnoreCase(allowDefaultJwtSigningKey)) { + log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); + } else { + String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; + log.error(message); + throw new ValidationException(message); + } + } + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 78e03ffe1a..a113f1e6d1 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -29,6 +30,7 @@ import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Customer; @@ -82,6 +84,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.config.JwtSettings; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -100,13 +103,18 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; +import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.TreeMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import static org.thingsboard.server.config.JwtSettings.ADMIN_SETTINGS_JWT_KEY; + @Service @Profile("install") @Slf4j @@ -167,6 +175,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private QueueService queueService; + @Autowired + private JwtSettings jwtSettings; + @Bean protected BCryptPasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(); @@ -656,4 +667,26 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { queueService.saveQueue(sequentialByOriginatorQueue); } } + + @Override + public void createJwtAdminSettings() throws Exception { + Objects.requireNonNull(jwtSettings,"JWT settings is null"); + AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + if (adminJwtSettings == null) { + if (jwtSettings.hasDefaultTokenSigningKey()) { + String allowDefaultJwtSigningKey = System.getenv("TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"); + if (!"true".equalsIgnoreCase(allowDefaultJwtSigningKey)) { + log.warn("JWT token signing key is default. Generating a new random key"); + jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); + } + } + adminJwtSettings = new AdminSettings(); + adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); + adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); + adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(jwtSettings)); + log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); + } + } + } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java index 1ceb1be289..4ef8241a3b 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java @@ -23,6 +23,8 @@ public interface SystemDataLoaderService { void createAdminSettings() throws Exception; + void createJwtAdminSettings() throws Exception; + void createOAuth2Templates() throws Exception; void loadSystemWidgets() throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index b90da586fc..862f0a92b8 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -159,6 +159,10 @@ public class DefaultDataUpdateService implements DataUpdateService { tenantsProfileQueueConfigurationUpdater.updateEntities(); rateLimitsUpdater.updateEntities(); break; + case "3.4.0": + log.info("Updating data from version 3.4.0 to 3.5.0 ..."); + systemDataLoaderService.createJwtAdminSettings(); + break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index a6e8637247..0e6a005e4d 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -107,7 +107,7 @@ plugins: # Security parameters security: # JWT Token parameters - jwt: + jwt: # Since 3.5.0 values are persisted to the database during install or upgrade. On Install, the key will be generated randomly if no custom value set. tokenExpirationTime: "${JWT_TOKEN_EXPIRATION_TIME:9000}" # Number of seconds (2.5 hours) refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:604800}" # Number of seconds (1 week) tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" From 7c8db6cac7245abcd42905c91864e2ece2d57c0b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 19 Sep 2022 15:44:28 +0300 Subject: [PATCH 03/80] jwt settings service implementation --- .../server/config/JwtSettings.java | 38 +----- .../server/config/JwtSettingsService.java | 117 ++++++++++++++++++ .../ConditionValidatorUpgradeServiceImpl.java | 31 +---- .../DefaultSystemDataLoaderService.java | 37 ++---- 4 files changed, 128 insertions(+), 95 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/JwtSettingsService.java diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java index 14e228bde8..b85f1738c9 100644 --- a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java +++ b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java @@ -15,27 +15,16 @@ */ package org.thingsboard.server.config; -import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.dao.settings.AdminSettingsService; - -import javax.annotation.PostConstruct; @Component @ConfigurationProperties(prefix = "security.jwt") @Data -@Slf4j public class JwtSettings { - public static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; - static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; + /** * {@link JwtToken} will expire after this time. */ @@ -57,29 +46,4 @@ public class JwtSettings { */ private Integer refreshTokenExpTime; - @JsonIgnore - @Autowired - private AdminSettingsService adminSettingsService; - - @PostConstruct - public void init() { - AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); - if (adminJwtSettings != null) { - log.debug("Loading the JWT admin settings from database"); - JwtSettings jwtSettings = JacksonUtil.treeToValue(adminJwtSettings.getJsonValue(), JwtSettings.class); - this.setRefreshTokenExpTime(jwtSettings.getRefreshTokenExpTime()); - this.setTokenExpirationTime(jwtSettings.getTokenExpirationTime()); - this.setTokenIssuer(jwtSettings.getTokenIssuer()); - this.setTokenSigningKey(jwtSettings.getTokenSigningKey()); - } - - if (hasDefaultTokenSigningKey()) { - log.warn("JWT token signing key is default. This is a security issue. Please, consider to set unique value"); - } - } - - public boolean hasDefaultTokenSigningKey() { - return TOKEN_SIGNING_KEY_DEFAULT.equals(tokenSigningKey); - } - } diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettingsService.java b/application/src/main/java/org/thingsboard/server/config/JwtSettingsService.java new file mode 100644 index 0000000000..bb7d3abc55 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/JwtSettingsService.java @@ -0,0 +1,117 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import javax.annotation.PostConstruct; +import javax.validation.ValidationException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Objects; + +@Service +@RequiredArgsConstructor +@Slf4j +public class JwtSettingsService { + + static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; + static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; + static final String TB_ALLOW_DEFAULT_JWT_SIGNING_KEY = "TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"; + + private final AdminSettingsService adminSettingsService; + + @Getter + private final JwtSettings jwtSettings; + + @PostConstruct + public void init() { + AdminSettings adminJwtSettings = findJwtAdminSettings(); + if (adminJwtSettings != null) { + log.debug("Loading the JWT admin settings from database"); + JwtSettings jwtLoaded = JacksonUtil.treeToValue(adminJwtSettings.getJsonValue(), JwtSettings.class); + jwtSettings.setRefreshTokenExpTime(jwtLoaded.getRefreshTokenExpTime()); + jwtSettings.setTokenExpirationTime(jwtLoaded.getTokenExpirationTime()); + jwtSettings.setTokenIssuer(jwtLoaded.getTokenIssuer()); + jwtSettings.setTokenSigningKey(jwtLoaded.getTokenSigningKey()); + } + + if (hasDefaultTokenSigningKey()) { + log.warn("JWT token signing key is default. This is a security issue. Please, consider to set unique value"); + } + } + + public boolean hasDefaultTokenSigningKey() { + return TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey()); + } + + public void createJwtAdminSettings() { + Objects.requireNonNull(jwtSettings, "JWT settings is null"); + if (!isJwtAdminSettingsExists()) { + if (hasDefaultTokenSigningKey()) { + if (!isAllowedDefaultJwtSigningKey()) { + log.warn("JWT token signing key is default. Generating a new random key"); + jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); + } + } + AdminSettings adminJwtSettings = new AdminSettings(); + adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); + adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); + adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(jwtSettings)); + log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); + } + } + + public boolean isJwtAdminSettingsExists() { + return findJwtAdminSettings() == null; + } + + AdminSettings findJwtAdminSettings() { + return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + } + + /* + * Allowing default JWT signing key is not secure + * */ + public boolean isAllowedDefaultJwtSigningKey() { + String allowDefaultJwtSigningKey = System.getenv(TB_ALLOW_DEFAULT_JWT_SIGNING_KEY); + return "true".equalsIgnoreCase(allowDefaultJwtSigningKey); + } + + public void validateJwtTokenSigningKey() { + if (!isJwtAdminSettingsExists()) { + if (hasDefaultTokenSigningKey()) { + if (isAllowedDefaultJwtSigningKey()) { + log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); + } else { + String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; + log.error(message); + throw new ValidationException(message); + } + } + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java index f70086abee..27117239f0 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java @@ -19,14 +19,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.config.JwtSettings; -import org.thingsboard.server.dao.settings.AdminSettingsService; - -import javax.validation.ValidationException; - -import static org.thingsboard.server.config.JwtSettings.ADMIN_SETTINGS_JWT_KEY; +import org.thingsboard.server.config.JwtSettingsService; @Service @Profile("install") @@ -34,30 +27,12 @@ import static org.thingsboard.server.config.JwtSettings.ADMIN_SETTINGS_JWT_KEY; @Slf4j public class ConditionValidatorUpgradeServiceImpl implements ConditionValidatorUpgradeService { - private final AdminSettingsService adminSettingsService; - - private final JwtSettings jwtSettings; + private final JwtSettingsService jwtSettingsService; @Override public void validateConditionsBeforeUpgrade(String fromVersion) throws Exception { log.info("Validating conditions before upgrade.."); - validateJwtTokenSigningKey(); - } - - void validateJwtTokenSigningKey() { - AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); - if (adminJwtSettings == null) { - if (jwtSettings.hasDefaultTokenSigningKey()) { - String allowDefaultJwtSigningKey = System.getenv("TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"); - if ("true".equalsIgnoreCase(allowDefaultJwtSigningKey)) { - log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); - } else { - String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; - log.error(message); - throw new ValidationException(message); - } - } - } + jwtSettingsService.validateJwtTokenSigningKey(); } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index a113f1e6d1..9c9ffb5188 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -22,7 +22,6 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; @@ -30,7 +29,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.context.annotation.Profile; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.Customer; @@ -84,7 +82,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.config.JwtSettingsService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -103,18 +101,13 @@ import org.thingsboard.server.dao.widget.WidgetsBundleService; import javax.annotation.Nullable; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; -import java.nio.charset.StandardCharsets; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.List; -import java.util.Objects; import java.util.TreeMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static org.thingsboard.server.config.JwtSettings.ADMIN_SETTINGS_JWT_KEY; - @Service @Profile("install") @Slf4j @@ -176,7 +169,7 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { private QueueService queueService; @Autowired - private JwtSettings jwtSettings; + private JwtSettingsService jwtSettingsService; @Bean protected BCryptPasswordEncoder passwordEncoder() { @@ -274,6 +267,11 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, mailSettings); } + @Override + public void createJwtAdminSettings() throws Exception { + jwtSettingsService.createJwtAdminSettings(); + } + @Override public void createOAuth2Templates() throws Exception { installScripts.createOAuth2Templates(); @@ -668,25 +666,4 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { } } - @Override - public void createJwtAdminSettings() throws Exception { - Objects.requireNonNull(jwtSettings,"JWT settings is null"); - AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); - if (adminJwtSettings == null) { - if (jwtSettings.hasDefaultTokenSigningKey()) { - String allowDefaultJwtSigningKey = System.getenv("TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"); - if (!"true".equalsIgnoreCase(allowDefaultJwtSigningKey)) { - log.warn("JWT token signing key is default. Generating a new random key"); - jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); - } - } - adminJwtSettings = new AdminSettings(); - adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); - adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); - adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(jwtSettings)); - log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); - } - } - } From 5ea3c9ff6a16d83c4f98e092670a23fdd0583a2b Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 19 Sep 2022 17:23:27 +0300 Subject: [PATCH 04/80] jwt settings service instead jwt settings data object --- .../server/config/JwtSettings.java | 1 - .../install/ThingsboardInstallService.java | 1 - .../security/auth/TokenOutdatingService.java | 6 ++--- .../security/model/token/JwtTokenFactory.java | 22 ++++++++----------- .../security/auth/JwtTokenFactoryTest.java | 8 ++++++- .../security/auth/TokenOutdatingTest.java | 10 +++++++-- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java index b85f1738c9..e5667dc811 100644 --- a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java +++ b/application/src/main/java/org/thingsboard/server/config/JwtSettings.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.security.model.JwtToken; @ConfigurationProperties(prefix = "security.jwt") @Data public class JwtSettings { - /** * {@link JwtToken} will expire after this time. */ diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index c83552e777..e8972696da 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -88,7 +88,6 @@ public class ThingsboardInstallService { @Autowired private ConditionValidatorUpgradeService conditionValidatorUpgradeService; - public void performInstall() { try { if (isUpgrade) { diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java index a623fc6862..73bb97ea0b 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.config.JwtSettingsService; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import javax.annotation.PostConstruct; @@ -39,7 +39,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; public class TokenOutdatingService { private final CacheManager cacheManager; private final JwtTokenFactory tokenFactory; - private final JwtSettings jwtSettings; + private final JwtSettingsService jwtSettingsService; private Cache usersUpdateTimeCache; @PostConstruct @@ -58,7 +58,7 @@ public class TokenOutdatingService { return Optional.ofNullable(usersUpdateTimeCache.get(toKey(userId), Long.class)) .map(outdatageTime -> { - if (System.currentTimeMillis() - outdatageTime <= SECONDS.toMillis(jwtSettings.getRefreshTokenExpTime())) { + if (System.currentTimeMillis() - outdatageTime <= SECONDS.toMillis(jwtSettingsService.getJwtSettings().getRefreshTokenExpTime())) { return MILLISECONDS.toSeconds(issueTime) < MILLISECONDS.toSeconds(outdatageTime); } else { /* diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index a8e6f9cb5c..a2fe509a1a 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -24,9 +24,9 @@ import io.jsonwebtoken.MalformedJwtException; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.SignatureException; import io.jsonwebtoken.UnsupportedJwtException; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.core.GrantedAuthority; import org.springframework.stereotype.Component; @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.config.JwtSettingsService; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; import org.thingsboard.server.service.security.model.JwtTokenPair; import org.thingsboard.server.service.security.model.SecurityUser; @@ -49,6 +49,7 @@ import java.util.UUID; import java.util.stream.Collectors; @Component +@RequiredArgsConstructor @Slf4j public class JwtTokenFactory { @@ -61,12 +62,7 @@ public class JwtTokenFactory { private static final String TENANT_ID = "tenantId"; private static final String CUSTOMER_ID = "customerId"; - private final JwtSettings settings; - - @Autowired - public JwtTokenFactory(JwtSettings settings) { - this.settings = settings; - } + private final JwtSettingsService jwtSettingsService; /** * Factory method for issuing new JWT Tokens. @@ -79,7 +75,7 @@ public class JwtTokenFactory { UserPrincipal principal = securityUser.getUserPrincipal(); JwtBuilder jwtBuilder = setUpToken(securityUser, securityUser.getAuthorities().stream() - .map(GrantedAuthority::getAuthority).collect(Collectors.toList()), settings.getTokenExpirationTime()); + .map(GrantedAuthority::getAuthority).collect(Collectors.toList()), jwtSettingsService.getJwtSettings().getTokenExpirationTime()); jwtBuilder.claim(FIRST_NAME, securityUser.getFirstName()) .claim(LAST_NAME, securityUser.getLastName()) .claim(ENABLED, securityUser.isEnabled()) @@ -138,7 +134,7 @@ public class JwtTokenFactory { public JwtToken createRefreshToken(SecurityUser securityUser) { UserPrincipal principal = securityUser.getUserPrincipal(); - String token = setUpToken(securityUser, Collections.singletonList(Authority.REFRESH_TOKEN.name()), settings.getRefreshTokenExpTime()) + String token = setUpToken(securityUser, Collections.singletonList(Authority.REFRESH_TOKEN.name()), jwtSettingsService.getJwtSettings().getRefreshTokenExpTime()) .claim(IS_PUBLIC, principal.getType() == UserPrincipal.Type.PUBLIC_ID) .setId(UUID.randomUUID().toString()).compact(); @@ -188,16 +184,16 @@ public class JwtTokenFactory { return Jwts.builder() .setClaims(claims) - .setIssuer(settings.getTokenIssuer()) + .setIssuer(jwtSettingsService.getJwtSettings().getTokenIssuer()) .setIssuedAt(Date.from(currentTime.toInstant())) .setExpiration(Date.from(currentTime.plusSeconds(expirationTime).toInstant())) - .signWith(SignatureAlgorithm.HS512, settings.getTokenSigningKey()); + .signWith(SignatureAlgorithm.HS512, jwtSettingsService.getJwtSettings().getTokenSigningKey()); } public Jws parseTokenClaims(JwtToken token) { try { return Jwts.parser() - .setSigningKey(settings.getTokenSigningKey()) + .setSigningKey(jwtSettingsService.getJwtSettings().getTokenSigningKey()) .parseClaimsJws(token.getToken()); } catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) { log.debug("Invalid JWT Token", ex); diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java index f865c9b5e0..c3223b920e 100644 --- a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java @@ -24,6 +24,7 @@ import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.config.JwtSettingsService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.AccessJwtToken; @@ -36,6 +37,8 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.mock; public class JwtTokenFactoryTest { @@ -50,7 +53,10 @@ public class JwtTokenFactoryTest { jwtSettings.setTokenExpirationTime((int) TimeUnit.HOURS.toSeconds(2)); jwtSettings.setRefreshTokenExpTime((int) TimeUnit.DAYS.toSeconds(7)); - tokenFactory = new JwtTokenFactory(jwtSettings); + JwtSettingsService jwtSettingsService = mock(JwtSettingsService.class); + willReturn(jwtSettings).given(jwtSettingsService).getJwtSettings(); + + tokenFactory = new JwtTokenFactory(jwtSettingsService); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java index f804d4dcfd..ea5c7f20e8 100644 --- a/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java @@ -27,6 +27,7 @@ import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent; import org.thingsboard.server.common.data.security.model.JwtToken; import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.config.JwtSettingsService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; @@ -50,6 +51,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.BDDMockito.willReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -71,10 +73,14 @@ public class TokenOutdatingTest { jwtSettings.setTokenExpirationTime((int) MINUTES.toSeconds(10)); jwtSettings.setRefreshTokenExpTime((int) DAYS.toSeconds(7)); jwtSettings.setTokenSigningKey("secret"); - tokenFactory = new JwtTokenFactory(jwtSettings); + + JwtSettingsService jwtSettingsService = mock(JwtSettingsService.class); + willReturn(jwtSettings).given(jwtSettingsService).getJwtSettings(); + + tokenFactory = new JwtTokenFactory(jwtSettingsService); cacheManager = new ConcurrentMapCacheManager(); - tokenOutdatingService = new TokenOutdatingService(cacheManager, tokenFactory, jwtSettings); + tokenOutdatingService = new TokenOutdatingService(cacheManager, tokenFactory, jwtSettingsService); tokenOutdatingService.initCache(); userId = new UserId(UUID.randomUUID()); From c313e1cf9cfd2fae66346c8137f64b1179b679e7 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 19 Sep 2022 19:09:24 +0300 Subject: [PATCH 05/80] jwt settings - running install on msa black box tests --- .../server/ThingsboardInstallApplication.java | 1 + .../server/config/{ => jwt}/JwtSettings.java | 2 +- .../config/{ => jwt}/JwtSettingsService.java | 33 +++++++++++-------- .../ConditionValidatorUpgradeServiceImpl.java | 4 +-- .../DefaultSystemDataLoaderService.java | 2 +- .../security/auth/TokenOutdatingService.java | 2 +- .../security/model/token/JwtTokenFactory.java | 2 +- .../security/auth/JwtTokenFactoryTest.java | 4 +-- .../security/auth/TokenOutdatingTest.java | 4 +-- packaging/java/scripts/install/logback.xml | 4 +++ 10 files changed, 34 insertions(+), 24 deletions(-) rename application/src/main/java/org/thingsboard/server/config/{ => jwt}/JwtSettings.java (96%) rename application/src/main/java/org/thingsboard/server/config/{ => jwt}/JwtSettingsService.java (74%) diff --git a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java index e90ed98351..6009dafafa 100644 --- a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java +++ b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java @@ -32,6 +32,7 @@ import java.util.Arrays; "org.thingsboard.server.dao", "org.thingsboard.server.common.stats", "org.thingsboard.server.common.transport.config.ssl", + "org.thingsboard.server.config.jwt", "org.thingsboard.server.cache", "org.thingsboard.server.springfox" }) diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java similarity index 96% rename from application/src/main/java/org/thingsboard/server/config/JwtSettings.java rename to application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java index e5667dc811..f99b36f32e 100644 --- a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config; +package org.thingsboard.server.config.jwt; import lombok.Data; import org.springframework.boot.context.properties.ConfigurationProperties; diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettingsService.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java similarity index 74% rename from application/src/main/java/org/thingsboard/server/config/JwtSettingsService.java rename to application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java index bb7d3abc55..bbb1bdba8b 100644 --- a/application/src/main/java/org/thingsboard/server/config/JwtSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config; +package org.thingsboard.server.config.jwt; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.dao.InvalidDataAccessResourceUsageException; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.AdminSettings; @@ -67,11 +68,12 @@ public class JwtSettingsService { } public void createJwtAdminSettings() { + log.debug("Creating JWT admin settings..."); Objects.requireNonNull(jwtSettings, "JWT settings is null"); - if (!isJwtAdminSettingsExists()) { + if (isJwtAdminSettingsNotExists()) { if (hasDefaultTokenSigningKey()) { if (!isAllowedDefaultJwtSigningKey()) { - log.warn("JWT token signing key is default. Generating a new random key"); + log.info("JWT token signing key is default. Generating a new random key"); jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); } } @@ -84,12 +86,17 @@ public class JwtSettingsService { } } - public boolean isJwtAdminSettingsExists() { + public boolean isJwtAdminSettingsNotExists() { return findJwtAdminSettings() == null; } AdminSettings findJwtAdminSettings() { - return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + try { + return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + } catch (InvalidDataAccessResourceUsageException ignored) { + log.debug("findAdminSettingsByKey is returning InvalidDataAccessResourceUsageException. This is an installation case when the database is not initialized yet"); + return null; + } } /* @@ -101,15 +108,13 @@ public class JwtSettingsService { } public void validateJwtTokenSigningKey() { - if (!isJwtAdminSettingsExists()) { - if (hasDefaultTokenSigningKey()) { - if (isAllowedDefaultJwtSigningKey()) { - log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); - } else { - String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; - log.error(message); - throw new ValidationException(message); - } + if (isJwtAdminSettingsNotExists() && hasDefaultTokenSigningKey()) { + if (isAllowedDefaultJwtSigningKey()) { + log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); + } else { + String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; + log.error(message); + throw new ValidationException(message); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java index 27117239f0..8dbfaab893 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java @@ -19,7 +19,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; -import org.thingsboard.server.config.JwtSettingsService; +import org.thingsboard.server.config.jwt.JwtSettingsService; @Service @Profile("install") @@ -31,7 +31,7 @@ public class ConditionValidatorUpgradeServiceImpl implements ConditionValidatorU @Override public void validateConditionsBeforeUpgrade(String fromVersion) throws Exception { - log.info("Validating conditions before upgrade.."); + log.info("Validating conditions before upgrade..."); jwtSettingsService.validateJwtTokenSigningKey(); } diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 9c9ffb5188..961d0265f9 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -82,7 +82,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.config.JwtSettingsService; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java index 73bb97ea0b..fb4cf74ee2 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/TokenOutdatingService.java @@ -25,7 +25,7 @@ import org.thingsboard.server.common.data.CacheConstants; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettingsService; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import javax.annotation.PostConstruct; diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index a2fe509a1a..cf19304ba7 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettingsService; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; import org.thingsboard.server.service.security.model.JwtTokenPair; import org.thingsboard.server.service.security.model.SecurityUser; diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java index c3223b920e..796ef93c0b 100644 --- a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java @@ -23,8 +23,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettings; -import org.thingsboard.server.config.JwtSettingsService; +import org.thingsboard.server.config.jwt.JwtSettings; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.AccessJwtToken; diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java index ea5c7f20e8..0ff639d66e 100644 --- a/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/TokenOutdatingTest.java @@ -26,8 +26,8 @@ import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.event.UserAuthDataChangedEvent; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettings; -import org.thingsboard.server.config.JwtSettingsService; +import org.thingsboard.server.config.jwt.JwtSettings; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.service.security.auth.jwt.JwtAuthenticationProvider; diff --git a/packaging/java/scripts/install/logback.xml b/packaging/java/scripts/install/logback.xml index 0047956c93..9233ab4d0b 100644 --- a/packaging/java/scripts/install/logback.xml +++ b/packaging/java/scripts/install/logback.xml @@ -56,6 +56,10 @@ + + + + From 5a11764cc47e65e8b115bdf390dc71f804e380b1 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko <79898499+smatvienko-tb@users.noreply.github.com> Date: Mon, 26 Sep 2022 11:47:53 +0300 Subject: [PATCH 06/80] PR template: Crosslinks between PRs added Having crosslinks between PRs is much easier to navigate between editions for reviewers and developers investigating the old PRs. --- pull_request_template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pull_request_template.md b/pull_request_template.md index 6e7c31967e..9d266abf7c 100644 --- a/pull_request_template.md +++ b/pull_request_template.md @@ -11,7 +11,7 @@ Put your PR description here instead of this sentence. - [ ] Description contains brief notes about what needs to be added to the documentation. - [ ] No merge conflicts, commented blocks of code, code formatting issues. - [ ] Changes are backward compatible or upgrade script is provided. -- [ ] Similar PR is opened for PE version to simplify merge. Required for internal contributors only. +- [ ] Similar PR is opened for PE version to simplify merge. Crosslinks between PRs added. Required for internal contributors only. ## Front-End feature checklist From ea80f9838e09384105899239bf22eeeb697171a8 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 1 Nov 2022 19:42:38 +0200 Subject: [PATCH 07/80] JwtSettings API added to the admin controller --- .../server/config/jwt/JwtSettingsService.java | 104 +----------- .../config/jwt/JwtSettingsServiceDefault.java | 154 ++++++++++++++++++ .../config/jwt/JwtSettingsValidator.java | 58 +++++++ .../server/controller/AdminController.java | 39 +++++ 4 files changed, 256 insertions(+), 99 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java create mode 100644 application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java index bbb1bdba8b..fb673fe50c 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java @@ -15,108 +15,14 @@ */ package org.thingsboard.server.config.jwt; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.dao.InvalidDataAccessResourceUsageException; -import org.springframework.stereotype.Service; -import org.thingsboard.common.util.JacksonUtil; -import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.common.data.id.TenantId; -import org.thingsboard.server.dao.settings.AdminSettingsService; +public interface JwtSettingsService { -import javax.annotation.PostConstruct; -import javax.validation.ValidationException; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.Objects; + JwtSettings getJwtSettings(); -@Service -@RequiredArgsConstructor -@Slf4j -public class JwtSettingsService { + void createJwtAdminSettings(); - static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; - static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; - static final String TB_ALLOW_DEFAULT_JWT_SIGNING_KEY = "TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"; + JwtSettings saveJwtSettings(JwtSettings jwtSettings); - private final AdminSettingsService adminSettingsService; - - @Getter - private final JwtSettings jwtSettings; - - @PostConstruct - public void init() { - AdminSettings adminJwtSettings = findJwtAdminSettings(); - if (adminJwtSettings != null) { - log.debug("Loading the JWT admin settings from database"); - JwtSettings jwtLoaded = JacksonUtil.treeToValue(adminJwtSettings.getJsonValue(), JwtSettings.class); - jwtSettings.setRefreshTokenExpTime(jwtLoaded.getRefreshTokenExpTime()); - jwtSettings.setTokenExpirationTime(jwtLoaded.getTokenExpirationTime()); - jwtSettings.setTokenIssuer(jwtLoaded.getTokenIssuer()); - jwtSettings.setTokenSigningKey(jwtLoaded.getTokenSigningKey()); - } - - if (hasDefaultTokenSigningKey()) { - log.warn("JWT token signing key is default. This is a security issue. Please, consider to set unique value"); - } - } - - public boolean hasDefaultTokenSigningKey() { - return TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey()); - } - - public void createJwtAdminSettings() { - log.debug("Creating JWT admin settings..."); - Objects.requireNonNull(jwtSettings, "JWT settings is null"); - if (isJwtAdminSettingsNotExists()) { - if (hasDefaultTokenSigningKey()) { - if (!isAllowedDefaultJwtSigningKey()) { - log.info("JWT token signing key is default. Generating a new random key"); - jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString(RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); - } - } - AdminSettings adminJwtSettings = new AdminSettings(); - adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); - adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); - adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(jwtSettings)); - log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); - adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); - } - } - - public boolean isJwtAdminSettingsNotExists() { - return findJwtAdminSettings() == null; - } - - AdminSettings findJwtAdminSettings() { - try { - return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); - } catch (InvalidDataAccessResourceUsageException ignored) { - log.debug("findAdminSettingsByKey is returning InvalidDataAccessResourceUsageException. This is an installation case when the database is not initialized yet"); - return null; - } - } - - /* - * Allowing default JWT signing key is not secure - * */ - public boolean isAllowedDefaultJwtSigningKey() { - String allowDefaultJwtSigningKey = System.getenv(TB_ALLOW_DEFAULT_JWT_SIGNING_KEY); - return "true".equalsIgnoreCase(allowDefaultJwtSigningKey); - } - - public void validateJwtTokenSigningKey() { - if (isJwtAdminSettingsNotExists() && hasDefaultTokenSigningKey()) { - if (isAllowedDefaultJwtSigningKey()) { - log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); - } else { - String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; - log.error(message); - throw new ValidationException(message); - } - } - } + void validateJwtTokenSigningKey(); } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java new file mode 100644 index 0000000000..5c172c9c2d --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java @@ -0,0 +1,154 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config.jwt; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.springframework.dao.InvalidDataAccessResourceUsageException; +import org.springframework.stereotype.Service; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.cluster.TbClusterService; +import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.id.TenantId; +import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.dao.settings.AdminSettingsService; + +import javax.annotation.PostConstruct; +import javax.validation.ValidationException; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Objects; + +@Service +@RequiredArgsConstructor +@Slf4j +public class JwtSettingsServiceDefault implements JwtSettingsService { + + static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; + static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; + static final String TB_ALLOW_DEFAULT_JWT_SIGNING_KEY = "TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"; + + private final AdminSettingsService adminSettingsService; + private final TbClusterService tbClusterService; + + private final JwtSettingsValidator jwtSettingsValidator; + + @Getter + private final JwtSettings jwtSettings; + + @PostConstruct + public void init() { + reloadJwtSettings(); + } + + void reloadJwtSettings() { + AdminSettings adminJwtSettings = findJwtAdminSettings(); + if (adminJwtSettings != null) { + log.debug("Loading the JWT admin settings from database"); + JwtSettings jwtLoaded = mapAdminToJwtSettings(adminJwtSettings); + jwtSettings.setRefreshTokenExpTime(jwtLoaded.getRefreshTokenExpTime()); + jwtSettings.setTokenExpirationTime(jwtLoaded.getTokenExpirationTime()); + jwtSettings.setTokenIssuer(jwtLoaded.getTokenIssuer()); + jwtSettings.setTokenSigningKey(jwtLoaded.getTokenSigningKey()); + } + + if (hasDefaultTokenSigningKey()) { + log.warn("JWT token signing key is default. This is a security issue. Please, consider to set unique value"); + } + } + + JwtSettings mapAdminToJwtSettings(AdminSettings adminSettings) { + Objects.requireNonNull(adminSettings, "adminSettings for JWT is null"); + return JacksonUtil.treeToValue(adminSettings.getJsonValue(), JwtSettings.class); + } + + AdminSettings mapJwtToAdminSettings(JwtSettings jwtSettings) { + Objects.requireNonNull(jwtSettings, "jwtSettings is null"); + AdminSettings adminJwtSettings = new AdminSettings(); + adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); + adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); + adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(jwtSettings)); + return adminJwtSettings; + } + + boolean hasDefaultTokenSigningKey() { + return TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey()); + } + + @Override + public void createJwtAdminSettings() { + log.debug("Creating JWT admin settings..."); + Objects.requireNonNull(jwtSettings, "JWT settings is null"); + if (isJwtAdminSettingsNotExists()) { + if (hasDefaultTokenSigningKey()) { + if (!isAllowedDefaultJwtSigningKey()) { + log.info("JWT token signing key is default. Generating a new random key"); + jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString( + RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); + } + } + saveJwtSettings(jwtSettings); + } + } + + @Override + public JwtSettings saveJwtSettings(JwtSettings jwtSettings){ + jwtSettingsValidator.validate(jwtSettings); + AdminSettings adminJwtSettings = mapJwtToAdminSettings(jwtSettings); + log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); + tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, ComponentLifecycleEvent.UPDATED); + reloadJwtSettings(); + return getJwtSettings(); + } + + boolean isJwtAdminSettingsNotExists() { + return findJwtAdminSettings() == null; + } + + AdminSettings findJwtAdminSettings() { + try { + return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + } catch (InvalidDataAccessResourceUsageException ignored) { + log.debug("findAdminSettingsByKey is returning InvalidDataAccessResourceUsageException. This is an installation case when the database is not initialized yet"); + return null; + } + } + + /* + * Allowing default JWT signing key is not secure + * */ + boolean isAllowedDefaultJwtSigningKey() { + String allowDefaultJwtSigningKey = System.getenv(TB_ALLOW_DEFAULT_JWT_SIGNING_KEY); + return "true".equalsIgnoreCase(allowDefaultJwtSigningKey); + } + + @Override + public void validateJwtTokenSigningKey() { + if (isJwtAdminSettingsNotExists() && hasDefaultTokenSigningKey()) { + if (isAllowedDefaultJwtSigningKey()) { + log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); + } else { + String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; + log.error(message); + throw new ValidationException(message); + } + } + } + +} diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java new file mode 100644 index 0000000000..4e91c654e0 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config.jwt; + +import lombok.AllArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.Arrays; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.Base64; +import java.util.Optional; + +@Component +@AllArgsConstructor +public class JwtSettingsValidator { + + public void validate(JwtSettings jwtSettings) { + if (StringUtils.isEmpty(jwtSettings.getTokenIssuer())) { + throw new DataValidationException("JWT token issuer should be specified!"); + } + if (Optional.ofNullable(jwtSettings.getRefreshTokenExpTime()).orElse(0) <= 0) { + throw new DataValidationException("JWT refresh token expiration time should be specified!"); + } + if (Optional.ofNullable(jwtSettings.getTokenExpirationTime()).orElse(0) <= 0) { + throw new DataValidationException("JWT token expiration time should be specified!"); + } + if (StringUtils.isEmpty(jwtSettings.getTokenSigningKey())) { + throw new DataValidationException("JWT token signing key should be specified!"); + } + + byte[] decodedKey; + try { + decodedKey = Base64.getDecoder().decode(jwtSettings.getTokenSigningKey()); + } catch (Exception e) { + throw new DataValidationException("JWT token signing key should be valid Base64 encoded string! " + e.getCause()); + } + + if (Arrays.isNullOrEmpty(decodedKey)) { + throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!"); + } + Arrays.fill(decodedKey, (byte) 0); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 54cbdd9b0f..09f552d0fd 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -23,6 +23,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import org.springframework.web.context.request.async.DeferredResult; @@ -36,6 +37,8 @@ import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; +import org.thingsboard.server.config.jwt.JwtSettings; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.permission.Operation; @@ -64,6 +67,9 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Autowired + private JwtSettingsService jwtSettingsService; + @Autowired private EntitiesVersionControlService versionControlService; @@ -151,6 +157,39 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get the JWT Settings object (getJwtSettings)", + notes = "Get the JWT Settings object that contains JWT token policy, etc. " + SYSTEM_AUTHORITY_PARAGRAPH, + produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @RequestMapping(value = "/jwtSettings", method = RequestMethod.GET) + @ResponseBody + public JwtSettings getJwtSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return checkNotNull(jwtSettingsService.getJwtSettings()); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Update JWT Settings (saveSecuritySettings)", + notes = "Updates the JWT Settings object that contains JWT token policy, etc. The tokenSigningKey field is a Base64 encoded string." + SYSTEM_AUTHORITY_PARAGRAPH, + produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @RequestMapping(value = "/jwtSettings", method = RequestMethod.POST) + @ResponseBody + public JwtSettings saveJwtSettings( + @ApiParam(value = "A JSON value representing the JWT Settings.") + @RequestBody JwtSettings jwtSettings) throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); + jwtSettings = checkNotNull(jwtSettingsService.saveJwtSettings(jwtSettings)); + return jwtSettings; + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Send test email (sendTestMail)", notes = "Attempts to send test email to the System Administrator User using Mail Settings provided as a parameter. " + "You may change the 'To' email in the user profile of the System Administrator. " + SYSTEM_AUTHORITY_PARAGRAPH) From a08c716ad28a01d2e226aaa7002e4818182162aa Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Tue, 1 Nov 2022 20:11:41 +0200 Subject: [PATCH 08/80] upgrade for JwtSetting targeted from 3.4.1 to 3.4.2 --- .../server/install/ThingsboardInstallService.java | 4 ---- .../service/install/update/DefaultDataUpdateService.java | 5 +---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index 52de6c91fb..10f6d38430 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -238,10 +238,6 @@ public class ThingsboardInstallService { log.info("Updating system data..."); systemDataLoaderService.updateSystemWidgets(); break; - case "3.4.0": - log.info("Upgrading ThingsBoard from version 3.4.0 to 3.5.0 ..."); - dataUpdateService.updateData("3.4.0"); - break; //TODO update CacheCleanupService on the next version upgrade diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 4909df84e8..d00d8f2768 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -181,6 +181,7 @@ public class DefaultDataUpdateService implements DataUpdateService { } break; case "3.4.1": + systemDataLoaderService.createJwtAdminSettings(); boolean skipAuditLogsMigration = getEnv("TB_SKIP_AUDIT_LOGS_MIGRATION", false); if (!skipAuditLogsMigration) { log.info("Updating data from version 3.4.1 to 3.4.2 ..."); @@ -190,10 +191,6 @@ public class DefaultDataUpdateService implements DataUpdateService { log.info("Skipping audit logs migration"); } break; - case "3.4.0": - log.info("Updating data from version 3.4.0 to 3.5.0 ..."); - systemDataLoaderService.createJwtAdminSettings(); - break; default: throw new RuntimeException("Unable to update data, unsupported fromVersion: " + fromVersion); } From 62d6ffd7157470ae09294c504aaaed696148757b Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 4 Nov 2022 18:28:54 +0200 Subject: [PATCH 09/80] Moved black-box tests to TestNg, added TestRestClient --- application/src/main/resources/logback.xml | 3 +- msa/black-box-tests/pom.xml | 29 +- .../server/msa/AbstractContainerTest.java | 110 ++------ .../server/msa/ContainerTestSuite.java | 221 ++++++++------- .../server/msa/TestProperties.java | 58 ++++ .../server/msa/TestRestClient.java | 253 ++++++++++++++++++ .../server/msa/ThingsBoardDbInstaller.java | 9 +- .../msa/connectivity/HttpClientTest.java | 114 +++----- .../msa/connectivity/MqttClientTest.java | 227 ++++++---------- .../connectivity/MqttGatewayClientTest.java | 230 +++++++--------- .../msa/prototypes/DevicePrototypes.java | 41 +++ .../src/test/resources/config.properties | 2 + 12 files changed, 750 insertions(+), 547 deletions(-) create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java create mode 100644 msa/black-box-tests/src/test/resources/config.properties diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 941bd7d278..90ed4785df 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,7 +25,8 @@ - + + diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index a1c745e163..054b4ee5a8 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -57,11 +57,6 @@ httpclient test - - io.takari.junit - takari-cpsuite - test - org.springframework.boot spring-boot-starter-test @@ -72,6 +67,30 @@ junit-vintage-engine test + + org.testng + testng + 7.6.1 + test + + + org.assertj + assertj-core + 3.23.1 + test + + + io.rest-assured + rest-assured + 5.2.0 + test + + + org.hamcrest + hamcrest-all + 1.3 + test + org.awaitility awaitility diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index 07529baead..df622a7752 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -15,13 +15,14 @@ */ package org.thingsboard.server.msa; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import io.restassured.RestAssured; +import io.restassured.filter.log.RequestLoggingFilter; +import io.restassured.filter.log.ResponseLoggingFilter; import lombok.extern.slf4j.Slf4j; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; @@ -33,97 +34,52 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.thingsboard.rest.client.RestClient; -import org.thingsboard.server.common.data.Device; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeSuite; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.msa.mapper.WsTelemetryResponse; import javax.net.ssl.SSLContext; import java.net.URI; -import java.util.List; -import java.util.Map; -import java.util.Random; +import java.util.*; @Slf4j public abstract class AbstractContainerTest { - protected static final String HTTPS_URL = "https://localhost"; - protected static final String WSS_URL = "wss://localhost"; - protected static String TB_TOKEN; - protected static RestClient restClient; - protected static long timeoutMultiplier = 1; protected ObjectMapper mapper = new ObjectMapper(); protected JsonParser jsonParser = new JsonParser(); - - @BeforeClass - public static void before() throws Exception { - restClient = new RestClient(HTTPS_URL); - restClient.getRestTemplate().setRequestFactory(getRequestFactoryForSelfSignedCert()); - + protected ContainerTestSuite containerTestSuite = ContainerTestSuite.getInstance(); + protected static TestRestClient testRestClient; + + @BeforeSuite + public void beforeSuite() { + RestAssured.filters(new RequestLoggingFilter(), new ResponseLoggingFilter()); + if ("false".equals(System.getProperty("runLocal", "false"))) { + containerTestSuite.start(); + } + testRestClient = new TestRestClient(TestProperties.getBaseUrl()); if (!"kafka".equals(System.getProperty("blackBoxTests.queue", "kafka"))) { timeoutMultiplier = 10; } } - @Rule - public TestRule watcher = new TestWatcher() { - protected void starting(Description description) { - log.info("================================================="); - log.info("STARTING TEST: {}" , description.getMethodName()); - log.info("================================================="); + @AfterSuite + public void afterSuite() { + if (containerTestSuite.isActive()) { + containerTestSuite.stop(); } - - /** - * Invoked when a test succeeds - */ - protected void succeeded(Description description) { - log.info("================================================="); - log.info("SUCCEEDED TEST: {}" , description.getMethodName()); - log.info("================================================="); - } - - /** - * Invoked when a test fails - */ - protected void failed(Throwable e, Description description) { - log.info("================================================="); - log.info("FAILED TEST: {}" , description.getMethodName(), e); - log.info("================================================="); - } - }; - - protected Device createGatewayDevice() throws JsonProcessingException { - String isGateway = "{\"gateway\":true}"; - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode additionalInfo = objectMapper.readTree(isGateway); - Device gatewayDeviceTemplate = new Device(); - gatewayDeviceTemplate.setName("mqtt_gateway"); - gatewayDeviceTemplate.setType("gateway"); - gatewayDeviceTemplate.setAdditionalInfo(additionalInfo); - return restClient.saveDevice(gatewayDeviceTemplate); - } - - protected Device createDevice(String name) { - Device device = new Device(); - device.setName(name + StringUtils.randomAlphanumeric(7)); - device.setType("DEFAULT"); - return restClient.saveDevice(device); } protected WsClient subscribeToWebSocket(DeviceId deviceId, String scope, CmdsType property) throws Exception { - WsClient wsClient = new WsClient(new URI(WSS_URL + "/api/ws/plugins/telemetry?token=" + restClient.getToken()), timeoutMultiplier); - SSLContextBuilder builder = SSLContexts.custom(); - builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true); - wsClient.setSocketFactory(builder.build().getSocketFactory()); + String webSocketUrl = TestProperties.getWebSocketUrl(); + WsClient wsClient = new WsClient(new URI(webSocketUrl + "/api/ws/plugins/telemetry?token=" + testRestClient.getToken()), timeoutMultiplier); + if (webSocketUrl.matches("^(wss)://.*$")) { + SSLContextBuilder builder = SSLContexts.custom(); + builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true); + wsClient.setSocketFactory(builder.build().getSocketFactory()); + } wsClient.connectBlocking(); JsonObject cmdsObject = new JsonObject(); @@ -150,16 +106,6 @@ public abstract class AbstractContainerTest { .build(); } - protected boolean verify(WsTelemetryResponse wsTelemetryResponse, String key, Long expectedTs, String expectedValue) { - List list = wsTelemetryResponse.getDataValuesByKey(key); - return expectedTs.equals(list.get(0)) && expectedValue.equals(list.get(1)); - } - - protected boolean verify(WsTelemetryResponse wsTelemetryResponse, String key, String expectedValue) { - List list = wsTelemetryResponse.getDataValuesByKey(key); - return expectedValue.equals(list.get(1)); - } - protected JsonObject createGatewayConnectPayload(String deviceName){ JsonObject payload = new JsonObject(); payload.addProperty("device", deviceName); @@ -216,7 +162,7 @@ public abstract class AbstractContainerTest { } } - private static HttpComponentsClientHttpRequestFactory getRequestFactoryForSelfSignedCert() throws Exception { + public static HttpComponentsClientHttpRequestFactory getRequestFactoryForSelfSignedCert() throws Exception { SSLContextBuilder builder = SSLContexts.custom(); builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true); SSLContext sslContext = builder.build(); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 47acbde16e..c32a6bdbe6 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -17,9 +17,6 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.junit.ClassRule; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.thingsboard.server.common.data.StringUtils; @@ -41,10 +38,8 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.testng.Assert.fail; -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) @Slf4j public class ContainerTestSuite { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); @@ -57,107 +52,133 @@ public class ContainerTestSuite { private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); - private static DockerComposeContainer testContainer; - - @ClassRule - public static ThingsBoardDbInstaller installTb = new ThingsBoardDbInstaller(); - - @ClassRule - public static DockerComposeContainer getTestContainer() { - if (testContainer == null) { - log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); - log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); - boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); - try { - final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; - log.info("targetDir {}", targetDir); - FileUtils.copyDirectory(new File(SOURCE_DIR), new File(targetDir)); - replaceInFile(targetDir + "docker-compose.yml", " container_name: \"${LOAD_BALANCER_NAME}\"", "", "container_name"); - - FileUtils.copyDirectory(new File("src/test/resources"), new File(targetDir)); - - class DockerComposeContainerImpl> extends DockerComposeContainer { - public DockerComposeContainerImpl(List composeFiles) { - super(composeFiles); - } - - @Override - public void stop() { - super.stop(); - tryDeleteDir(targetDir); - } - } + private DockerComposeContainer testContainer; + private ThingsBoardDbInstaller installTb; + public boolean isActive; + + private static ContainerTestSuite containerTestSuite; + + public boolean isActive() { + return isActive; + } + + public void setActive(boolean active) { + isActive = active; + } + + private ContainerTestSuite() { + } + + public static ContainerTestSuite getInstance() { + if (containerTestSuite == null) { + containerTestSuite = new ContainerTestSuite(); + } + return containerTestSuite; + } + + public void start() { + installTb = new ThingsBoardDbInstaller(); + installTb.createVolumes(); + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); + log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); + boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); + try { + final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; + log.info("targetDir {}", targetDir); + FileUtils.copyDirectory(new File(SOURCE_DIR), new File(targetDir)); + replaceInFile(targetDir + "docker-compose.yml", " container_name: \"${LOAD_BALANCER_NAME}\"", "", "container_name"); - List composeFiles = new ArrayList<>(Arrays.asList( - new File(targetDir + "docker-compose.yml"), - new File(targetDir + "docker-compose.volumes.yml"), - new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), - new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), - new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), - new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")) - )); - - Map queueEnv = new HashMap<>(); - queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE); - switch (QUEUE_TYPE) { - case "kafka": - composeFiles.add(new File(targetDir + "docker-compose.kafka.yml")); - break; - case "aws-sqs": - replaceInFile(targetDir, "queue-aws-sqs.env", - Map.of("YOUR_KEY", getSysProp("blackBoxTests.awsKey"), - "YOUR_SECRET", getSysProp("blackBoxTests.awsSecret"), - "YOUR_REGION", getSysProp("blackBoxTests.awsRegion"))); - break; - case "rabbitmq": - composeFiles.add(new File(targetDir + "docker-compose.rabbitmq-server.yml")); - replaceInFile(targetDir, "queue-rabbitmq.env", - Map.of("localhost", "rabbitmq")); - break; - case "service-bus": - replaceInFile(targetDir, "queue-service-bus.env", - Map.of("YOUR_NAMESPACE_NAME", getSysProp("blackBoxTests.serviceBusNamespace"), - "YOUR_SAS_KEY_NAME", getSysProp("blackBoxTests.serviceBusSASPolicy"))); - replaceInFile(targetDir, "queue-service-bus.env", - Map.of("YOUR_SAS_KEY", getSysProp("blackBoxTests.serviceBusPrimaryKey"))); - break; - case "pubsub": - replaceInFile(targetDir, "queue-pubsub.env", - Map.of("YOUR_PROJECT_ID", getSysProp("blackBoxTests.pubSubProjectId"), - "YOUR_SERVICE_ACCOUNT", getSysProp("blackBoxTests.pubSubServiceAccount"))); - break; - default: - throw new RuntimeException("Unsupported queue type: " + QUEUE_TYPE); + FileUtils.copyDirectory(new File("src/test/resources"), new File(targetDir)); + + class DockerComposeContainerImpl> extends DockerComposeContainer { + public DockerComposeContainerImpl(List composeFiles) { + super(composeFiles); } - if (IS_HYBRID_MODE) { - composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); + @Override + public void stop() { + super.stop(); + tryDeleteDir(targetDir); } + } - testContainer = new DockerComposeContainerImpl<>(composeFiles) - .withPull(false) - .withLocalCompose(true) - .withTailChildContainers(!skipTailChildContainers) - .withEnv(installTb.getEnv()) - .withEnv(queueEnv) - .withEnv("LOAD_BALANCER_NAME", "") - .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); - } catch (Exception e) { - log.error("Failed to create test container", e); - fail("Failed to create test container"); + List composeFiles = new ArrayList<>(Arrays.asList( + new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.volumes.yml"), + new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), + new File(targetDir + "docker-compose.postgres.volumes.yml"), + new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), + new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), + new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")) + )); + + Map queueEnv = new HashMap<>(); + queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE); + switch (QUEUE_TYPE) { + case "kafka": + composeFiles.add(new File(targetDir + "docker-compose.kafka.yml")); + break; + case "aws-sqs": + replaceInFile(targetDir, "queue-aws-sqs.env", + Map.of("YOUR_KEY", getSysProp("blackBoxTests.awsKey"), + "YOUR_SECRET", getSysProp("blackBoxTests.awsSecret"), + "YOUR_REGION", getSysProp("blackBoxTests.awsRegion"))); + break; + case "rabbitmq": + composeFiles.add(new File(targetDir + "docker-compose.rabbitmq-server.yml")); + replaceInFile(targetDir, "queue-rabbitmq.env", + Map.of("localhost", "rabbitmq")); + break; + case "service-bus": + replaceInFile(targetDir, "queue-service-bus.env", + Map.of("YOUR_NAMESPACE_NAME", getSysProp("blackBoxTests.serviceBusNamespace"), + "YOUR_SAS_KEY_NAME", getSysProp("blackBoxTests.serviceBusSASPolicy"))); + replaceInFile(targetDir, "queue-service-bus.env", + Map.of("YOUR_SAS_KEY", getSysProp("blackBoxTests.serviceBusPrimaryKey"))); + break; + case "pubsub": + replaceInFile(targetDir, "queue-pubsub.env", + Map.of("YOUR_PROJECT_ID", getSysProp("blackBoxTests.pubSubProjectId"), + "YOUR_SERVICE_ACCOUNT", getSysProp("blackBoxTests.pubSubServiceAccount"))); + break; + default: + throw new RuntimeException("Unsupported queue type: " + QUEUE_TYPE); } + + if (IS_HYBRID_MODE) { + composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); + } + + testContainer = new DockerComposeContainerImpl<>(composeFiles) + .withPull(false) + .withLocalCompose(true) + .withTailChildContainers(!skipTailChildContainers) + .withEnv(installTb.getEnv()) + .withEnv(queueEnv) + .withEnv("LOAD_BALANCER_NAME", "") + .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + testContainer.start(); + setActive(true); + } catch (Exception e) { + log.error("Failed to create test container", e); + fail("Failed to create test container"); + } + } + public void stop() { + if (isActive) { + testContainer.stop(); + installTb.savaLogsAndRemoveVolumes(); + setActive(false); } - return testContainer; } private static void replaceInFile(String targetDir, String fileName, Map replacements) throws IOException { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java new file mode 100644 index 0000000000..f50caa1aee --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -0,0 +1,58 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +public class TestProperties { + public static final String HTTPS_URL = "https://localhost"; + + protected static final String WSS_URL = "wss://localhost"; + + public static final ContainerTestSuite instance = ContainerTestSuite.getInstance(); + + public static String getBaseUrl(){ + if (instance.isActive()) { + return HTTPS_URL; + } + return getProperty("tb.baseUrl"); + } + + public static String getWebSocketUrl(){ + if (instance.isActive()) { + return WSS_URL; + } + return getProperty("tb.baseUrl"); + } + + private static String getProperty(String propertyName) { + + try (InputStream input = TestProperties.class.getClassLoader().getResourceAsStream("config.properties")) { + Properties prop = new Properties(); + prop.load(input); + return prop.getProperty(propertyName); + + } catch (IOException ex) { + ex.printStackTrace(); + } + return null; + + } + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java new file mode 100644 index 0000000000..eaf5332323 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -0,0 +1,253 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import com.fasterxml.jackson.databind.JsonNode; +import io.restassured.common.mapper.TypeRef; +import io.restassured.http.ContentType; +import io.restassured.path.json.JsonPath; +import io.restassured.response.ValidatableResponse; +import io.restassured.specification.RequestSpecification; +import org.springframework.http.HttpStatus; +import org.thingsboard.rest.client.RestClient; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.security.DeviceCredentials; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static io.restassured.RestAssured.given; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.core.AnyOf.anyOf; +import static org.thingsboard.server.common.data.StringUtils.isEmpty; +import static org.thingsboard.server.msa.AbstractContainerTest.getRequestFactoryForSelfSignedCert; + +public class TestRestClient { + private static final String JWT_TOKEN_HEADER_PARAM = "X-Authorization"; + private final String baseURL; + private String token; + private String refreshToken; + private RequestSpecification spec; + protected static final String ACTIVATE_TOKEN_REGEX = "/api/noauth/activate?activateToken="; + + public TestRestClient(String url) { + baseURL = url; + spec = given().baseUri(baseURL).contentType(ContentType.JSON); + if (url.matches("^(https)://.*$")) { + spec.relaxedHTTPSValidation(); + } + } + + public void login(String username, String password) throws Exception { + Map loginRequest = new HashMap<>(); + loginRequest.put("username", username); + loginRequest.put("password", password); + + JsonPath jsonPath = given().relaxedHTTPSValidation().body(loginRequest).post(baseURL + "/api/auth/login") + .getBody().jsonPath(); + token = jsonPath.get("token"); + refreshToken = jsonPath.get("refreshToken"); + spec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token) + .contentType(ContentType.JSON); + } + + public Device postDevice(String accessToken, Device device) { + return given().spec(spec).body(device) + .pathParams("accessToken", accessToken) + .post("/api/device?accessToken={accessToken}") + .then() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(Device.class); + } + + public ValidatableResponse getDeviceById(DeviceId deviceId, int statusCode) { + return given().spec(spec) + .pathParams("deviceId", deviceId.getId()) + .get("/api/device/{deviceId}") + .then() + .statusCode(statusCode); + } + public Device getDeviceById(DeviceId deviceId) { + return getDeviceById(deviceId, HttpStatus.OK.value()) + .extract() + .as(Device.class); + } + public DeviceCredentials getDeviceCredentialsByDeviceId(DeviceId deviceId) { + return given().spec(spec).get("/api/device/{deviceId}/credentials", deviceId.getId()) + .then() + .assertThat() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(DeviceCredentials.class); + } + + public ValidatableResponse postTelemetry(String credentialsId, JsonNode telemetry) { + return given().spec(spec).body(telemetry) + .post("/api/v1/{credentialsId}/telemetry", credentialsId) + .then() + .statusCode(HttpStatus.OK.value()); + } + + public ValidatableResponse deleteDevice(DeviceId deviceId) { + return given().spec(spec) + .delete("/api/device/{deviceId}", deviceId.getId()) + .then() + .statusCode(HttpStatus.OK.value()); + } + public ValidatableResponse deleteDeviceIfExists(DeviceId deviceId) { + return given().spec(spec) + .delete("/api/device/{deviceId}", deviceId.getId()) + .then() + .statusCode(anyOf(is(HttpStatus.OK.value()),is(HttpStatus.NOT_FOUND.value()))); + } + + public ValidatableResponse postTelemetryAttribute(String entityType, DeviceId deviceId, String scope, JsonNode attribute) { + return given().spec(spec).body(attribute) + .post("/api/plugins/telemetry/{entityType}/{entityId}/attributes/{scope}", entityType, deviceId.getId(), scope) + .then() + .statusCode(HttpStatus.OK.value()); + } + + public ValidatableResponse postAttribute(String accessToken, JsonNode attribute) { + return given().spec(spec).body(attribute) + .post("/api/v1/{accessToken}/attributes/", accessToken) + .then() + .statusCode(HttpStatus.OK.value()); + } + + public JsonNode getAttributes(String accessToken, String clientKeys, String sharedKeys) { + return given().spec(spec) + .queryParam("clientKeys", clientKeys) + .queryParam("sharedKeys", sharedKeys) + .get("/api/v1/{accessToken}/attributes", accessToken) + .then() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(JsonNode.class); + } + + public PageData getRuleChains(PageLink pageLink) { + Map params = new HashMap<>(); + addPageLinkToParam(params, pageLink); + return given().spec(spec).queryParams(params) + .get("/api/ruleChains") + .then() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(new TypeRef>() {}); + } + + public RuleChain postRootRuleChain(RuleChain ruleChain) { + return given().spec(spec) + .body(ruleChain) + .post("/api/ruleChain") + .then() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(RuleChain.class); + } + + public RuleChainMetaData postRuleChainMetadata(RuleChainMetaData ruleChainMetaData) { + return given().spec(spec) + .body(ruleChainMetaData) + .post("/api/ruleChain/metadata") + .then() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(RuleChainMetaData.class); + } + + public void setRootRuleChain(RuleChainId ruleChainId) { + given().spec(spec) + .post("/api/ruleChain/{ruleChainId}/root", ruleChainId.getId()) + .then() + .statusCode(HttpStatus.OK.value()); + } + + public void deleteRuleChain(RuleChainId ruleChainId) { + given().spec(spec) + .delete("/api/ruleChain/{ruleChainId}", ruleChainId.getId()) + .then() + .statusCode(HttpStatus.OK.value()); + } + + private String getUrlParams(PageLink pageLink) { + String urlParams = "pageSize={pageSize}&page={page}"; + if (!isEmpty(pageLink.getTextSearch())) { + urlParams += "&textSearch={textSearch}"; + } + if (pageLink.getSortOrder() != null) { + urlParams += "&sortProperty={sortProperty}&sortOrder={sortOrder}"; + } + return urlParams; + } + + private void addPageLinkToParam(Map params, PageLink pageLink) { + params.put("pageSize", String.valueOf(pageLink.getPageSize())); + params.put("page", String.valueOf(pageLink.getPage())); + if (!isEmpty(pageLink.getTextSearch())) { + params.put("textSearch", pageLink.getTextSearch()); + } + if (pageLink.getSortOrder() != null) { + params.put("sortProperty", pageLink.getSortOrder().getProperty()); + params.put("sortOrder", pageLink.getSortOrder().getDirection().name()); + } + } + + public List findRelationByFrom(EntityId fromId, RelationTypeGroup relationTypeGroup) { + Map params = new HashMap<>(); + params.put("fromId", fromId.getId().toString()); + params.put("fromType", fromId.getEntityType().name()); + params.put("relationTypeGroup", relationTypeGroup.name()); + + return given().spec(spec) + .pathParams(params) + .get("/api/relations?fromId={fromId}&fromType={fromType}&relationTypeGroup={relationTypeGroup}") + .then() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(new TypeRef>() {}); + } + + public JsonNode postServerSideRpc(DeviceId deviceId, JsonNode serverRpcPayload) { + return given().spec(spec) + .body(serverRpcPayload) + .post("/api/rpc/twoway/{deviceId}", deviceId.getId()) + .then() + .statusCode(HttpStatus.OK.value()) + .extract() + .as(JsonNode.class); + } + + public String getToken() { + return token; + } + + public String getRefreshToken() { + return refreshToken; + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 979fbb187d..ed606cd468 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -16,7 +16,6 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; -import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import org.thingsboard.server.common.data.StringUtils; @@ -30,7 +29,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; @Slf4j -public class ThingsBoardDbInstaller extends ExternalResource { +public class ThingsBoardDbInstaller { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); @@ -129,8 +128,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { return env; } - @Override - protected void before() throws Throwable { + public void createVolumes() { try { dockerCompose.withCommand("volume create " + postgresDataVolume); @@ -192,8 +190,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { } } - @Override - protected void after() { + public void savaLogsAndRemoveVolumes() { copyLogs(tbLogVolume, "./target/tb-logs/"); copyLogs(tbCoapTransportLogVolume, "./target/tb-coap-transport-logs/"); copyLogs(tbLwm2mTransportLogVolume, "./target/tb-lwm2m-transport-logs/"); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java index f81d03b394..6f24f00854 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java @@ -16,10 +16,9 @@ package org.thingsboard.server.msa.connectivity; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Sets; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.http.ResponseEntity; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.msa.AbstractContainerTest; @@ -27,98 +26,69 @@ import org.thingsboard.server.msa.WsClient; import org.thingsboard.server.msa.mapper.WsTelemetryResponse; -import java.util.Optional; +import java.util.Arrays; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.thingsboard.server.common.data.DataConstants.DEVICE; -import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; public class HttpClientTest extends AbstractContainerTest { + private Device device; + @BeforeMethod + public void setUp() throws Exception { + testRestClient.login("tenant@thingsboard.org", "tenant"); + device = testRestClient.postDevice("", defaultDevicePrototype("http_")); + } + + @AfterMethod + public void tearDown() { + testRestClient.deleteDeviceIfExists(device.getId()); + } @Test public void telemetryUpload() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - - Device device = createDevice("http_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); - ResponseEntity deviceTelemetryResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/v1/{credentialsId}/telemetry", - mapper.readTree(createPayload().toString()), - ResponseEntity.class, - deviceCredentials.getCredentialsId()); - Assert.assertTrue(deviceTelemetryResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetry(deviceCredentials.getCredentialsId(), mapper.readTree(createPayload().toString())); + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); wsClient.closeBlocking(); - Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73))); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - restClient.deleteDevice(device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void getAttributes() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - TB_TOKEN = restClient.getToken(); - - Device device = createDevice("test"); - String accessToken = restClient.getDeviceCredentialsByDeviceId(device.getId()).get().getCredentialsId(); - assertNotNull(accessToken); + String accessToken = testRestClient.getDeviceCredentialsByDeviceId(device.getId()).getCredentialsId(); + assertThat(accessToken).isNotNull(); - ResponseEntity deviceSharedAttributes = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/" + DEVICE + "/" + device.getId().toString() + "/attributes/" + SHARED_SCOPE, mapper.readTree(createPayload().toString()), - ResponseEntity.class, - accessToken); + JsonNode sharedAattribute = mapper.readTree(createPayload().toString()); + testRestClient.postTelemetryAttribute(DEVICE, device.getId(), SHARED_SCOPE, sharedAattribute); - Assert.assertTrue(deviceSharedAttributes.getStatusCode().is2xxSuccessful()); - - ResponseEntity deviceClientsAttributes = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/v1/" + accessToken + "/attributes/", mapper.readTree(createPayload().toString()), - ResponseEntity.class, - accessToken); - - Assert.assertTrue(deviceClientsAttributes.getStatusCode().is2xxSuccessful()); + JsonNode clientAttribute = mapper.readTree(createPayload().toString()); + testRestClient.postAttribute(accessToken, clientAttribute); TimeUnit.SECONDS.sleep(3 * timeoutMultiplier); - @SuppressWarnings("deprecation") - Optional allOptional = restClient.getAttributes(accessToken, null, null); - assertTrue(allOptional.isPresent()); - - - JsonNode all = allOptional.get(); - assertEquals(2, all.size()); - assertEquals(mapper.readTree(createPayload().toString()), all.get("shared")); - assertEquals(mapper.readTree(createPayload().toString()), all.get("client")); - - @SuppressWarnings("deprecation") - Optional sharedOptional = restClient.getAttributes(accessToken, null, "stringKey"); - assertTrue(sharedOptional.isPresent()); - - JsonNode shared = sharedOptional.get(); - assertEquals(shared.get("shared").get("stringKey"), mapper.readTree(createPayload().get("stringKey").toString())); - assertFalse(shared.has("client")); + JsonNode attributes = testRestClient.getAttributes(accessToken, null, null); + assertThat(attributes.get("shared")).isEqualTo(sharedAattribute); + assertThat(attributes.get("client")).isEqualTo(clientAttribute); - @SuppressWarnings("deprecation") - Optional clientOptional = restClient.getAttributes(accessToken, "longKey,stringKey", null); - assertTrue(clientOptional.isPresent()); + JsonNode attributes2 = testRestClient.getAttributes(accessToken, null, "stringKey"); + assertThat(attributes2.get("shared").get("stringKey")).isEqualTo(sharedAattribute.get("stringKey")); + assertThat(attributes2.has("client")).isFalse(); - JsonNode client = clientOptional.get(); - assertFalse(client.has("shared")); - assertEquals(mapper.readTree(createPayload().get("longKey").toString()), client.get("client").get("longKey")); - assertEquals(client.get("client").get("stringKey"), mapper.readTree(createPayload().get("stringKey").toString())); + JsonNode attributes3 = testRestClient.getAttributes(accessToken, "longKey,stringKey", null); - restClient.deleteDevice(device.getId()); + assertThat(attributes3.has("shared")).isFalse(); + assertThat(attributes3.get("client").get("longKey")).isEqualTo(clientAttribute.get("longKey")); + assertThat(attributes3.get("client").get("stringKey")).isEqualTo(clientAttribute.get("stringKey")); } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index ce36f08a64..7e7ebf07ab 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.msa.connectivity; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -26,19 +25,19 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.NodeConnectionInfo; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -61,14 +60,30 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.fail; +import static org.thingsboard.server.common.data.DataConstants.DEVICE; +import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; +import static org.thingsboard.server.msa.TestProperties.HTTPS_URL; +import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; + @Slf4j public class MqttClientTest extends AbstractContainerTest { + private Device device; + @BeforeMethod + public void setUp() throws Exception { + testRestClient.login("tenant@thingsboard.org", "tenant"); + device = testRestClient.postDevice("", defaultDevicePrototype("http_")); + } + + @AfterMethod + public void tearDown() { + testRestClient.deleteDeviceIfExists(device.getId()); + } @Test public void telemetryUpload() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); MqttClient mqttClient = getMqttClient(deviceCredentials, null); @@ -77,25 +92,19 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"), - actualLatestTelemetry.getLatestValues().keySet()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73))); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void telemetryUploadWithTs() throws Exception { long ts = 1451649600512L; - - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); MqttClient mqttClient = getMqttClient(deviceCredentials, null); @@ -104,22 +113,18 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(getExpectedLatestValues(ts), actualLatestTelemetry.getLatestValues()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(getExpectedLatestValues(ts)).isEqualTo(actualLatestTelemetry.getLatestValues()); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", ts, Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", ts, "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", ts, Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", ts, Long.toString(73))); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void publishAttributeUpdateToServer() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); MqttMessageListener listener = new MqttMessageListener(); @@ -134,23 +139,18 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("attr1", "attr2", "attr3", "attr4"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "attr1", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "attr2", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "attr3", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "attr4", Long.toString(73))); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("attr1", "attr2", "attr3", "attr4")); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr1").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr2").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr3").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr4").get(1)).isEqualTo(Long.toString(73)); } @Test public void requestAttributeValuesFromServer() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); MqttMessageListener listener = new MqttMessageListener(); @@ -166,21 +166,16 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received ws telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(1, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("clientAttr"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "clientAttr", clientAttributeValue)); + assertThat(actualLatestTelemetry.getData()).hasSize(1); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnly("clientAttr"); + assertThat(actualLatestTelemetry.getDataValuesByKey("clientAttr").get(1)).isEqualTo(clientAttributeValue); // Add a new shared attribute JsonObject sharedAttributes = new JsonObject(); String sharedAttributeValue = StringUtils.randomAlphanumeric(8); sharedAttributes.addProperty("sharedAttr", sharedAttributeValue); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - device.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + JsonNode sharedAttribute = mapper.readTree(sharedAttributes.toString()); + testRestClient.postTelemetryAttribute(DataConstants.DEVICE, device.getId(), SHARED_SCOPE, sharedAttribute); // Subscribe to attributes response mqttClient.on("v1/devices/me/attributes/response/+", listener, MqttQoS.AT_LEAST_ONCE).get(); @@ -197,20 +192,16 @@ public class MqttClientTest extends AbstractContainerTest { AttributesResponse attributes = mapper.readValue(Objects.requireNonNull(event).getMessage(), AttributesResponse.class); log.info("Received telemetry: {}", attributes); - Assert.assertEquals(1, attributes.getClient().size()); - Assert.assertEquals(clientAttributeValue, attributes.getClient().get("clientAttr")); - - Assert.assertEquals(1, attributes.getShared().size()); - Assert.assertEquals(sharedAttributeValue, attributes.getShared().get("sharedAttr")); + assertThat(attributes.getClient()).hasSize(1); + assertThat(attributes.getClient().get("clientAttr")).isEqualTo(clientAttributeValue); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(attributes.getShared()).hasSize(1); + assertThat(attributes.getShared().get("sharedAttr")).isEqualTo(sharedAttributeValue); } @Test public void subscribeToAttributeUpdatesFromServer() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); @@ -225,38 +216,28 @@ public class MqttClientTest extends AbstractContainerTest { JsonObject sharedAttributes = new JsonObject(); String sharedAttributeValue = StringUtils.randomAlphanumeric(8); sharedAttributes.addProperty(sharedAttributeName, sharedAttributeValue); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - device.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + JsonNode sharedAttribute = mapper.readTree(sharedAttributes.toString()); + + testRestClient.postTelemetryAttribute(DataConstants.DEVICE, device.getId(), SHARED_SCOPE, sharedAttribute); MqttEvent event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(sharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()) + .isEqualTo(sharedAttributeValue); // Update the shared attribute value JsonObject updatedSharedAttributes = new JsonObject(); String updatedSharedAttributeValue = StringUtils.randomAlphanumeric(8); updatedSharedAttributes.addProperty(sharedAttributeName, updatedSharedAttributeValue); - ResponseEntity updatedSharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(updatedSharedAttributes.toString()), ResponseEntity.class, - device.getId()); - Assert.assertTrue(updatedSharedAttributesResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetryAttribute(DEVICE, device.getId(), SHARED_SCOPE, mapper.readTree(updatedSharedAttributes.toString())); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(updatedSharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()) + .isEqualTo(updatedSharedAttributeValue); } @Test public void serverSideRpc() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); @@ -270,21 +251,18 @@ public class MqttClientTest extends AbstractContainerTest { serverRpcPayload.addProperty("method", "getValue"); serverRpcPayload.addProperty("params", true); ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(getClass().getSimpleName()))); - ListenableFuture future = service.submit(() -> { + ListenableFuture future = service.submit(() -> { try { - return restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/rpc/twoway/{deviceId}", - mapper.readTree(serverRpcPayload.toString()), String.class, - device.getId()); + return testRestClient.postServerSideRpc(device.getId(), mapper.readTree(serverRpcPayload.toString())); } catch (IOException e) { - return ResponseEntity.badRequest().build(); + return null; } }); // Wait for RPC call from the server and send the response MqttEvent requestFromServer = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals("{\"method\":\"getValue\",\"params\":true}", Objects.requireNonNull(requestFromServer).getMessage()); + assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}"); Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); JsonObject clientResponse = new JsonObject(); @@ -292,19 +270,14 @@ public class MqttClientTest extends AbstractContainerTest { // Send a response to the server's RPC request mqttClient.publish("v1/devices/me/rpc/response/" + requestId, Unpooled.wrappedBuffer(clientResponse.toString().getBytes())).get(); - ResponseEntity serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); + JsonNode serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); service.shutdownNow(); - Assert.assertTrue(serverResponse.getStatusCode().is2xxSuccessful()); - Assert.assertEquals(clientResponse.toString(), serverResponse.getBody()); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(serverResponse).isEqualTo(mapper.readTree(clientResponse.toString())); } @Test public void clientSideRpc() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); @@ -328,46 +301,33 @@ public class MqttClientTest extends AbstractContainerTest { TimeUnit.SECONDS.sleep(1 * timeoutMultiplier); MqttEvent responseFromServer = listener.getEvents().poll(1 * timeoutMultiplier, TimeUnit.SECONDS); Integer responseId = Integer.valueOf(Objects.requireNonNull(responseFromServer).getTopic().substring("v1/devices/me/rpc/response/".length())); - Assert.assertEquals(requestId, responseId); - Assert.assertEquals("requestReceived", mapper.readTree(responseFromServer.getMessage()).get("response").asText()); + assertThat(responseId).isEqualTo(requestId); + assertThat(mapper.readTree(responseFromServer.getMessage()).get("response").asText()).isEqualTo("requestReceived"); // Make the default rule chain a root again - ResponseEntity rootRuleChainResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain/{ruleChainId}/root", - null, - RuleChain.class, - defaultRuleChainId); - Assert.assertTrue(rootRuleChainResponse.getStatusCode().is2xxSuccessful()); + testRestClient.setRootRuleChain(defaultRuleChainId); // Delete the created rule chain - restClient.getRestTemplate().delete(HTTPS_URL + "/api/ruleChain/{ruleChainId}", ruleChainId); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + testRestClient.deleteRuleChain(ruleChainId); } @Test public void deviceDeletedClosingSession() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - String deviceForDeletingTestName = "Device for deleting notification test"; - Device device = createDevice(deviceForDeletingTestName); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); - restClient.deleteDevice(device.getId()); + testRestClient.deleteDeviceIfExists(device.getId()); TimeUnit.SECONDS.sleep(3 * timeoutMultiplier); - Assert.assertFalse(mqttClient.isConnected()); + assertThat(mqttClient.isConnected()).isFalse(); } private RuleChainId createRootRuleChainForRpcResponse() throws Exception { RuleChain newRuleChain = new RuleChain(); newRuleChain.setName("testRuleChain"); - ResponseEntity ruleChainResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain", - newRuleChain, - RuleChain.class); - Assert.assertTrue(ruleChainResponse.getStatusCode().is2xxSuccessful()); - RuleChain ruleChain = ruleChainResponse.getBody(); + + RuleChain ruleChain = testRestClient.postRootRuleChain(newRuleChain); JsonNode configuration = mapper.readTree(this.getClass().getClassLoader().getResourceAsStream("RpcResponseRuleChainMetadata.json")); RuleChainMetaData ruleChainMetaData = new RuleChainMetaData(); @@ -376,37 +336,22 @@ public class MqttClientTest extends AbstractContainerTest { ruleChainMetaData.setNodes(Arrays.asList(mapper.treeToValue(configuration.get("nodes"), RuleNode[].class))); ruleChainMetaData.setConnections(Arrays.asList(mapper.treeToValue(configuration.get("connections"), NodeConnectionInfo[].class))); - ResponseEntity ruleChainMetadataResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain/metadata", - ruleChainMetaData, - RuleChainMetaData.class); - Assert.assertTrue(ruleChainMetadataResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postRuleChainMetadata(ruleChainMetaData); // Set a new rule chain as root - ResponseEntity rootRuleChainResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain/{ruleChainId}/root", - null, - RuleChain.class, - ruleChain.getId()); - Assert.assertTrue(rootRuleChainResponse.getStatusCode().is2xxSuccessful()); - + testRestClient.setRootRuleChain(ruleChain.getId()); return ruleChain.getId(); } private RuleChainId getDefaultRuleChainId() { - ResponseEntity> ruleChains = restClient.getRestTemplate().exchange( - HTTPS_URL + "/api/ruleChains?pageSize=40&page=0&textSearch=", - HttpMethod.GET, - null, - new ParameterizedTypeReference>() { - }); - - Optional defaultRuleChain = ruleChains.getBody().getData() + PageData ruleChains = testRestClient.getRuleChains(new PageLink(40, 0)); + + Optional defaultRuleChain = ruleChains.getData() .stream() .filter(RuleChain::isRoot) .findFirst(); if (!defaultRuleChain.isPresent()) { - Assert.fail("Root rule chain wasn't found"); + fail("Root rule chain wasn't found"); } return defaultRuleChain.get().getId(); } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index 25a8946431..f844be8a27 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -16,10 +16,6 @@ package org.thingsboard.server.msa.connectivity; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Sets; -import com.google.common.util.concurrent.ListenableFuture; -import com.google.common.util.concurrent.ListeningExecutorService; -import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -28,15 +24,15 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.springframework.http.ResponseEntity; -import org.thingsboard.common.util.ThingsBoardThreadFactory; +import org.springframework.http.HttpStatus; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; @@ -48,40 +44,40 @@ import org.thingsboard.server.msa.AbstractContainerTest; import org.thingsboard.server.msa.WsClient; import org.thingsboard.server.msa.mapper.WsTelemetryResponse; -import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Random; +import java.util.*; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutionException; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; +import static org.thingsboard.server.common.data.DataConstants.DEVICE; +import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; +import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultGatewayPrototype; + @Slf4j public class MqttGatewayClientTest extends AbstractContainerTest { - Device gatewayDevice; - MqttClient mqttClient; - Device createdDevice; - MqttMessageListener listener; + private Device gatewayDevice; + private MqttClient mqttClient; + private Device createdDevice; + private MqttMessageListener listener; - @Before + @BeforeMethod public void createGateway() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - this.gatewayDevice = createGatewayDevice(); - Optional gatewayDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(gatewayDevice.getId()); - Assert.assertTrue(gatewayDeviceCredentials.isPresent()); + testRestClient.login("tenant@thingsboard.org", "tenant"); + gatewayDevice = testRestClient.postDevice("", defaultGatewayPrototype()); + DeviceCredentials gatewayDeviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(gatewayDevice.getId()); + this.listener = new MqttMessageListener(); - this.mqttClient = getMqttClient(gatewayDeviceCredentials.get(), listener); + this.mqttClient = getMqttClient(gatewayDeviceCredentials, listener); this.createdDevice = createDeviceThroughGateway(mqttClient, gatewayDevice); } - @After - public void removeGateway() throws Exception { - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.gatewayDevice.getId()); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.createdDevice.getId()); + @AfterMethod + public void removeGateway() { + testRestClient.deleteDeviceIfExists(this.gatewayDevice.getId()); + testRestClient.deleteDeviceIfExists(this.createdDevice.getId()); this.listener = null; this.mqttClient = null; this.createdDevice = null; @@ -95,40 +91,38 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"), - actualLatestTelemetry.getLatestValues().keySet()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73))); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void telemetryUploadWithTs() throws Exception { long ts = 1451649600512L; - restClient.login("tenant@thingsboard.org", "tenant"); WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); mqttClient.publish("v1/gateway/telemetry", Unpooled.wrappedBuffer(createGatewayPayload(createdDevice.getName(), ts).toString().getBytes())).get(); WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(getExpectedLatestValues(ts), actualLatestTelemetry.getLatestValues()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", ts, Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", ts, "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", ts, Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", ts, Long.toString(73))); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void publishAttributeUpdateToServer() throws Exception { - Optional createdDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); - Assert.assertTrue(createdDeviceCredentials.isPresent()); + testRestClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); + WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); JsonObject clientAttributes = new JsonObject(); clientAttributes.addProperty("attr1", "value1"); @@ -142,20 +136,18 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info("Received attributes: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("attr1", "attr2", "attr3", "attr4"), - actualLatestTelemetry.getLatestValues().keySet()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("attr1", "attr2", "attr3", "attr4")); - Assert.assertTrue(verify(actualLatestTelemetry, "attr1", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "attr2", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "attr3", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "attr4", Long.toString(73))); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr1").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr2").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr3").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr4").get(1)).isEqualTo(Long.toString(73)); } @Test public void responseDataOnAttributesRequestCheck() throws Exception { - Optional createdDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); - Assert.assertTrue(createdDeviceCredentials.isPresent()); + testRestClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); JsonObject sharedAttributes = new JsonObject(); sharedAttributes.addProperty("attr1", "value1"); sharedAttributes.addProperty("attr2", true); @@ -163,11 +155,8 @@ public class MqttGatewayClientTest extends AbstractContainerTest { sharedAttributes.addProperty("attr4", 73); mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + + testRestClient.postTelemetryAttribute(DataConstants.DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(sharedAttributes.toString())); var event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); JsonObject requestData = new JsonObject(); @@ -181,8 +170,8 @@ public class MqttGatewayClientTest extends AbstractContainerTest { event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); JsonObject responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); - Assert.assertTrue(responseData.has("value")); - Assert.assertEquals(sharedAttributes.get("attr1").getAsString(), responseData.get("value").getAsString()); + assertThat(responseData.has("value")).isTrue(); + assertThat(responseData.get("value").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); requestData = new JsonObject(); requestData.addProperty("id", 1); @@ -198,9 +187,9 @@ public class MqttGatewayClientTest extends AbstractContainerTest { event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); - Assert.assertTrue(responseData.has("values")); - Assert.assertEquals(sharedAttributes.get("attr1").getAsString(), responseData.get("values").getAsJsonObject().get("attr1").getAsString()); - Assert.assertEquals(sharedAttributes.get("attr2").getAsString(), responseData.get("values").getAsJsonObject().get("attr2").getAsString()); + assertThat(responseData.has("value")).isTrue(); + assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); + assertThat(responseData.get("values").getAsJsonObject().get("attr2").getAsString()).isEqualTo(sharedAttributes.get("attr2").getAsString()); requestData = new JsonObject(); requestData.addProperty("id", 1); @@ -216,9 +205,9 @@ public class MqttGatewayClientTest extends AbstractContainerTest { event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); - Assert.assertTrue(responseData.has("values")); - Assert.assertEquals(sharedAttributes.get("attr1").getAsString(), responseData.get("values").getAsJsonObject().get("attr1").getAsString()); - Assert.assertEquals(1, responseData.get("values").getAsJsonObject().entrySet().size()); + assertThat(responseData.has("values")).isTrue(); + assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); + assertThat(responseData.get("values").getAsJsonObject().entrySet()).hasSize(1); } @Test @@ -237,11 +226,9 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info("Received ws telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(1, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("clientAttr"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "clientAttr", clientAttributeValue)); + assertThat(actualLatestTelemetry.getData()).hasSize(1); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnly("clientAttr"); + assertThat(actualLatestTelemetry.getDataValuesByKey("clientAttr").get(1)).isEqualTo(clientAttributeValue); // Add a new shared attribute JsonObject sharedAttributes = new JsonObject(); @@ -251,16 +238,12 @@ public class MqttGatewayClientTest extends AbstractContainerTest { // Subscribe for attribute update event mqttClient.on("v1/gateway/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetryAttribute(DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(sharedAttributes.toString())); MqttEvent sharedAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); // Catch attribute update event - Assert.assertNotNull(sharedAttributeEvent); - Assert.assertEquals("v1/gateway/attributes", sharedAttributeEvent.getTopic()); + assertThat(sharedAttributeEvent).isNotNull(); + assertThat(sharedAttributeEvent.getTopic()).isEqualTo("v1/gateway/attributes"); // Subscribe to attributes response mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); @@ -288,15 +271,11 @@ public class MqttGatewayClientTest extends AbstractContainerTest { gatewaySharedAttributeValue.addProperty("device", createdDevice.getName()); gatewaySharedAttributeValue.add("data", sharedAttributes); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetryAttribute(DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(sharedAttributes.toString())); MqttEvent event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(sharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()) + .isEqualTo(sharedAttributeValue); // Update the shared attribute value JsonObject updatedSharedAttributes = new JsonObject(); @@ -307,15 +286,10 @@ public class MqttGatewayClientTest extends AbstractContainerTest { gatewayUpdatedSharedAttributeValue.addProperty("device", createdDevice.getName()); gatewayUpdatedSharedAttributeValue.add("data", updatedSharedAttributes); - ResponseEntity updatedSharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(updatedSharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(updatedSharedAttributesResponse.getStatusCode().is2xxSuccessful()); - + testRestClient.postTelemetryAttribute(DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(updatedSharedAttributes.toString())); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(updatedSharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()) + .isEqualTo(updatedSharedAttributeValue); } @Test @@ -330,35 +304,18 @@ public class MqttGatewayClientTest extends AbstractContainerTest { JsonObject serverRpcPayload = new JsonObject(); serverRpcPayload.addProperty("method", "getValue"); serverRpcPayload.addProperty("params", true); - ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(getClass().getSimpleName()))); - ListenableFuture future = service.submit(() -> { - try { - return restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/rpc/twoway/{deviceId}", - mapper.readTree(serverRpcPayload.toString()), String.class, - createdDevice.getId()); - } catch (IOException e) { - return ResponseEntity.badRequest().build(); - } - }); + + JsonNode response = testRestClient.postServerSideRpc(createdDevice.getId(), mapper.readTree(serverRpcPayload.toString())); // Wait for RPC call from the server and send the response MqttEvent requestFromServer = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - service.shutdownNow(); - - Assert.assertNotNull(requestFromServer); - Assert.assertNotNull(requestFromServer.getMessage()); - - JsonObject requestFromServerJson = new JsonParser().parse(requestFromServer.getMessage()).getAsJsonObject(); - - Assert.assertEquals(createdDevice.getName(), requestFromServerJson.get("device").getAsString()); - - JsonObject requestFromServerData = requestFromServerJson.get("data").getAsJsonObject(); - - Assert.assertEquals("getValue", requestFromServerData.get("method").getAsString()); - Assert.assertTrue(requestFromServerData.get("params").getAsBoolean()); - - int requestId = requestFromServerData.get("id").getAsInt(); + assertThat(requestFromServer).isNotNull(); + assertThat(requestFromServer.getMessage()).isNotNull(); + JsonNode requestFromServerJson = JacksonUtil.toJsonNode(requestFromServer.getMessage()); + assertThat(requestFromServerJson.get("device").asText()).isEqualTo(createdDevice.getName()); + assertThat(requestFromServerJson.get("data").get("method").asText()).isEqualTo("getValue"); + assertThat(requestFromServerJson.get("data").get("params").asText()).isEqualTo("true"); + int requestId = requestFromServerJson.get("data").get("id").asInt(); JsonObject clientResponse = new JsonObject(); clientResponse.addProperty("response", "someResponse"); @@ -369,16 +326,14 @@ public class MqttGatewayClientTest extends AbstractContainerTest { // Send a response to the server's RPC request mqttClient.publish(gatewayRpcTopic, Unpooled.wrappedBuffer(gatewayResponse.toString().getBytes())).get(); - ResponseEntity serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertTrue(serverResponse.getStatusCode().is2xxSuccessful()); - Assert.assertEquals(clientResponse.toString(), serverResponse.getBody()); + + assertThat(response).isEqualTo(clientResponse.getAsJsonObject()); } @Test public void deviceCreationAfterDeleted() throws Exception { - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.createdDevice.getId()); - Optional deletedDevice = restClient.getDeviceById(this.createdDevice.getId()); - Assert.assertTrue(deletedDevice.isEmpty()); + testRestClient.deleteDevice(this.createdDevice.getId()); + testRestClient.getDeviceById(this.createdDevice.getId(), HttpStatus.NOT_FOUND.value()); this.createdDevice = createDeviceThroughGateway(mqttClient, gatewayDevice); } @@ -397,13 +352,13 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info(gatewayAttributesRequest.toString()); mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get(); MqttEvent clientAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertNotNull(clientAttributeEvent); + assertThat(clientAttributeEvent).isNotNull(); JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject(); - Assert.assertEquals(messageId, responseMessage.get("id").getAsInt()); - Assert.assertEquals(createdDevice.getName(), responseMessage.get("device").getAsString()); - Assert.assertEquals(3, responseMessage.entrySet().size()); - Assert.assertEquals(expectedValue, responseMessage.get("value").getAsString()); + assertThat(responseMessage.get("id").getAsInt()).isEqualTo(messageId); + assertThat(responseMessage.get("device").getAsString()).isEqualTo(createdDevice.getName()); + assertThat(responseMessage.entrySet()).hasSize(3); + assertThat(responseMessage.get("value").getAsString()).isEqualTo(expectedValue); } private Device createDeviceThroughGateway(MqttClient mqttClient, Device gatewayDevice) throws Exception { @@ -418,17 +373,12 @@ public class MqttGatewayClientTest extends AbstractContainerTest { TimeUnit.SECONDS.sleep(30); } - List relations = restClient.findByFrom(gatewayDevice.getId(), RelationTypeGroup.COMMON); - - Assert.assertEquals(1, relations.size()); + List relations = testRestClient.findRelationByFrom(gatewayDevice.getId(), RelationTypeGroup.COMMON); + assertThat(relations).hasSize(1); EntityId createdEntityId = relations.get(0).getTo(); DeviceId createdDeviceId = new DeviceId(createdEntityId.getId()); - Optional createdDevice = restClient.getDeviceById(createdDeviceId); - - Assert.assertTrue(createdDevice.isPresent()); - - return createdDevice.get(); + return testRestClient.getDeviceById(createdDeviceId); } private MqttClient getMqttClient(DeviceCredentials deviceCredentials, MqttMessageListener listener) throws InterruptedException, ExecutionException { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java new file mode 100644 index 0000000000..d7edcb6699 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java @@ -0,0 +1,41 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.prototypes; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Device; + +public class DevicePrototypes { + public static Device defaultDevicePrototype(String name){ + Device device = new Device(); + device.setName(name + RandomStringUtils.randomAlphanumeric(7)); + device.setType("DEFAULT"); + return device; + } + + public static Device defaultGatewayPrototype() throws JsonProcessingException { + String isGateway = "{\"gateway\":true}"; + JsonNode additionalInfo = JacksonUtil.valueToTree(isGateway); + Device gatewayDeviceTemplate = new Device(); + gatewayDeviceTemplate.setName("mqtt_gateway " + RandomStringUtils.randomAlphabetic(5)); + gatewayDeviceTemplate.setType("gateway"); + gatewayDeviceTemplate.setAdditionalInfo(additionalInfo); + return gatewayDeviceTemplate; + } +} diff --git a/msa/black-box-tests/src/test/resources/config.properties b/msa/black-box-tests/src/test/resources/config.properties new file mode 100644 index 0000000000..419c73185d --- /dev/null +++ b/msa/black-box-tests/src/test/resources/config.properties @@ -0,0 +1,2 @@ +tb.baseUrl=http://localhost:8080 +tb.wsUrl=ws://localhost:8080 From c5e1c5a1a51330d6d510685564b2f1beb7a50f7e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 7 Nov 2022 13:19:17 +0200 Subject: [PATCH 10/80] test fixes, refactoring --- msa/black-box-tests/pom.xml | 4 -- .../server/msa/AbstractContainerTest.java | 1 - .../server/msa/TestProperties.java | 2 +- .../server/msa/TestRestClient.java | 62 ++++++++++++------- .../connectivity/MqttGatewayClientTest.java | 32 +++++++--- .../msa/prototypes/DevicePrototypes.java | 3 +- pom.xml | 28 +++++++++ 7 files changed, 93 insertions(+), 39 deletions(-) diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 054b4ee5a8..9cd38c0c05 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -70,25 +70,21 @@ org.testng testng - 7.6.1 test org.assertj assertj-core - 3.23.1 test io.rest-assured rest-assured - 5.2.0 test org.hamcrest hamcrest-all - 1.3 test diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index df622a7752..df1fe4a522 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -55,7 +55,6 @@ public abstract class AbstractContainerTest { @BeforeSuite public void beforeSuite() { - RestAssured.filters(new RequestLoggingFilter(), new ResponseLoggingFilter()); if ("false".equals(System.getProperty("runLocal", "false"))) { containerTestSuite.start(); } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java index f50caa1aee..6e3a024cda 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -38,7 +38,7 @@ public class TestProperties { if (instance.isActive()) { return WSS_URL; } - return getProperty("tb.baseUrl"); + return getProperty("tb.wsUrl"); } private static String getProperty(String propertyName) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index eaf5332323..0b0c4b89bc 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -16,13 +16,21 @@ package org.thingsboard.server.msa; import com.fasterxml.jackson.databind.JsonNode; +import io.restassured.RestAssured; +import io.restassured.builder.ResponseSpecBuilder; import io.restassured.common.mapper.TypeRef; +import io.restassured.config.HeaderConfig; +import io.restassured.config.HttpClientConfig; +import io.restassured.config.RestAssuredConfig; +import io.restassured.filter.log.RequestLoggingFilter; +import io.restassured.filter.log.ResponseLoggingFilter; import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; +import io.restassured.specification.ResponseSpecification; +import org.hamcrest.Matchers; import org.springframework.http.HttpStatus; -import org.thingsboard.rest.client.RestClient; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -40,24 +48,33 @@ import java.util.List; import java.util.Map; import static io.restassured.RestAssured.given; +import static org.apache.http.params.CoreConnectionPNames.CONNECTION_TIMEOUT; +import static org.apache.http.params.CoreConnectionPNames.SO_TIMEOUT; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.AnyOf.anyOf; import static org.thingsboard.server.common.data.StringUtils.isEmpty; -import static org.thingsboard.server.msa.AbstractContainerTest.getRequestFactoryForSelfSignedCert; public class TestRestClient { private static final String JWT_TOKEN_HEADER_PARAM = "X-Authorization"; private final String baseURL; private String token; private String refreshToken; - private RequestSpecification spec; + private RequestSpecification requestSpec; + private ResponseSpecification responseSpec; protected static final String ACTIVATE_TOKEN_REGEX = "/api/noauth/activate?activateToken="; public TestRestClient(String url) { + RestAssured.filters(new RequestLoggingFilter(), new ResponseLoggingFilter()); + baseURL = url; - spec = given().baseUri(baseURL).contentType(ContentType.JSON); + requestSpec = given().baseUri(baseURL) + .contentType(ContentType.JSON) + .config(RestAssuredConfig.config() + .headerConfig(HeaderConfig.headerConfig() + .overwriteHeadersWithName(JWT_TOKEN_HEADER_PARAM))); + if (url.matches("^(https)://.*$")) { - spec.relaxedHTTPSValidation(); + requestSpec.relaxedHTTPSValidation(); } } @@ -70,12 +87,11 @@ public class TestRestClient { .getBody().jsonPath(); token = jsonPath.get("token"); refreshToken = jsonPath.get("refreshToken"); - spec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token) - .contentType(ContentType.JSON); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); } public Device postDevice(String accessToken, Device device) { - return given().spec(spec).body(device) + return given().spec(requestSpec).body(device) .pathParams("accessToken", accessToken) .post("/api/device?accessToken={accessToken}") .then() @@ -85,7 +101,7 @@ public class TestRestClient { } public ValidatableResponse getDeviceById(DeviceId deviceId, int statusCode) { - return given().spec(spec) + return given().spec(requestSpec) .pathParams("deviceId", deviceId.getId()) .get("/api/device/{deviceId}") .then() @@ -97,7 +113,7 @@ public class TestRestClient { .as(Device.class); } public DeviceCredentials getDeviceCredentialsByDeviceId(DeviceId deviceId) { - return given().spec(spec).get("/api/device/{deviceId}/credentials", deviceId.getId()) + return given().spec(requestSpec).get("/api/device/{deviceId}/credentials", deviceId.getId()) .then() .assertThat() .statusCode(HttpStatus.OK.value()) @@ -106,41 +122,41 @@ public class TestRestClient { } public ValidatableResponse postTelemetry(String credentialsId, JsonNode telemetry) { - return given().spec(spec).body(telemetry) + return given().spec(requestSpec).body(telemetry) .post("/api/v1/{credentialsId}/telemetry", credentialsId) .then() .statusCode(HttpStatus.OK.value()); } public ValidatableResponse deleteDevice(DeviceId deviceId) { - return given().spec(spec) + return given().spec(requestSpec) .delete("/api/device/{deviceId}", deviceId.getId()) .then() .statusCode(HttpStatus.OK.value()); } public ValidatableResponse deleteDeviceIfExists(DeviceId deviceId) { - return given().spec(spec) + return given().spec(requestSpec) .delete("/api/device/{deviceId}", deviceId.getId()) .then() .statusCode(anyOf(is(HttpStatus.OK.value()),is(HttpStatus.NOT_FOUND.value()))); } public ValidatableResponse postTelemetryAttribute(String entityType, DeviceId deviceId, String scope, JsonNode attribute) { - return given().spec(spec).body(attribute) + return given().spec(requestSpec).body(attribute) .post("/api/plugins/telemetry/{entityType}/{entityId}/attributes/{scope}", entityType, deviceId.getId(), scope) .then() .statusCode(HttpStatus.OK.value()); } public ValidatableResponse postAttribute(String accessToken, JsonNode attribute) { - return given().spec(spec).body(attribute) + return given().spec(requestSpec).body(attribute) .post("/api/v1/{accessToken}/attributes/", accessToken) .then() .statusCode(HttpStatus.OK.value()); } public JsonNode getAttributes(String accessToken, String clientKeys, String sharedKeys) { - return given().spec(spec) + return given().spec(requestSpec) .queryParam("clientKeys", clientKeys) .queryParam("sharedKeys", sharedKeys) .get("/api/v1/{accessToken}/attributes", accessToken) @@ -153,7 +169,7 @@ public class TestRestClient { public PageData getRuleChains(PageLink pageLink) { Map params = new HashMap<>(); addPageLinkToParam(params, pageLink); - return given().spec(spec).queryParams(params) + return given().spec(requestSpec).queryParams(params) .get("/api/ruleChains") .then() .statusCode(HttpStatus.OK.value()) @@ -162,7 +178,7 @@ public class TestRestClient { } public RuleChain postRootRuleChain(RuleChain ruleChain) { - return given().spec(spec) + return given().spec(requestSpec) .body(ruleChain) .post("/api/ruleChain") .then() @@ -172,7 +188,7 @@ public class TestRestClient { } public RuleChainMetaData postRuleChainMetadata(RuleChainMetaData ruleChainMetaData) { - return given().spec(spec) + return given().spec(requestSpec) .body(ruleChainMetaData) .post("/api/ruleChain/metadata") .then() @@ -182,14 +198,14 @@ public class TestRestClient { } public void setRootRuleChain(RuleChainId ruleChainId) { - given().spec(spec) + given().spec(requestSpec) .post("/api/ruleChain/{ruleChainId}/root", ruleChainId.getId()) .then() .statusCode(HttpStatus.OK.value()); } public void deleteRuleChain(RuleChainId ruleChainId) { - given().spec(spec) + given().spec(requestSpec) .delete("/api/ruleChain/{ruleChainId}", ruleChainId.getId()) .then() .statusCode(HttpStatus.OK.value()); @@ -224,7 +240,7 @@ public class TestRestClient { params.put("fromType", fromId.getEntityType().name()); params.put("relationTypeGroup", relationTypeGroup.name()); - return given().spec(spec) + return given().spec(requestSpec) .pathParams(params) .get("/api/relations?fromId={fromId}&fromType={fromType}&relationTypeGroup={relationTypeGroup}") .then() @@ -234,7 +250,7 @@ public class TestRestClient { } public JsonNode postServerSideRpc(DeviceId deviceId, JsonNode serverRpcPayload) { - return given().spec(spec) + return given().spec(requestSpec) .body(serverRpcPayload) .post("/api/rpc/twoway/{deviceId}", deviceId.getId()) .then() diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index f844be8a27..54aec26421 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -16,6 +16,9 @@ package org.thingsboard.server.msa.connectivity; import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -24,11 +27,15 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; @@ -44,12 +51,10 @@ import org.thingsboard.server.msa.AbstractContainerTest; import org.thingsboard.server.msa.WsClient; import org.thingsboard.server.msa.mapper.WsTelemetryResponse; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.*; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import static org.assertj.core.api.Assertions.assertThat; import static org.thingsboard.server.common.data.DataConstants.DEVICE; @@ -187,7 +192,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); - assertThat(responseData.has("value")).isTrue(); + assertThat(responseData.has("values")).isTrue(); assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); assertThat(responseData.get("values").getAsJsonObject().get("attr2").getAsString()).isEqualTo(sharedAttributes.get("attr2").getAsString()); @@ -304,11 +309,19 @@ public class MqttGatewayClientTest extends AbstractContainerTest { JsonObject serverRpcPayload = new JsonObject(); serverRpcPayload.addProperty("method", "getValue"); serverRpcPayload.addProperty("params", true); - - JsonNode response = testRestClient.postServerSideRpc(createdDevice.getId(), mapper.readTree(serverRpcPayload.toString())); + ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(getClass().getSimpleName()))); + ListenableFuture future = service.submit(() -> { + try { + return testRestClient.postServerSideRpc(createdDevice.getId(), mapper.readTree(serverRpcPayload.toString())); + } catch (IOException e) { + return null; + } + }); // Wait for RPC call from the server and send the response MqttEvent requestFromServer = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); + service.shutdownNow(); + assertThat(requestFromServer).isNotNull(); assertThat(requestFromServer.getMessage()).isNotNull(); JsonNode requestFromServerJson = JacksonUtil.toJsonNode(requestFromServer.getMessage()); @@ -326,8 +339,9 @@ public class MqttGatewayClientTest extends AbstractContainerTest { // Send a response to the server's RPC request mqttClient.publish(gatewayRpcTopic, Unpooled.wrappedBuffer(gatewayResponse.toString().getBytes())).get(); + JsonNode serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); - assertThat(response).isEqualTo(clientResponse.getAsJsonObject()); + assertThat(serverResponse).isEqualTo(mapper.readTree(clientResponse.toString())); } @Test @@ -366,7 +380,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { TimeUnit.SECONDS.sleep(30); } - String deviceName = "mqtt_device"; + String deviceName = "mqtt_device" + RandomStringUtils.randomAlphabetic(5); mqttClient.publish("v1/gateway/connect", Unpooled.wrappedBuffer(createGatewayConnectPayload(deviceName).toString().getBytes()), MqttQoS.AT_LEAST_ONCE).get(); if (timeoutMultiplier > 1) { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java index d7edcb6699..e8ed09df18 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java @@ -17,6 +17,7 @@ package org.thingsboard.server.msa.prototypes; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; @@ -33,7 +34,7 @@ public class DevicePrototypes { String isGateway = "{\"gateway\":true}"; JsonNode additionalInfo = JacksonUtil.valueToTree(isGateway); Device gatewayDeviceTemplate = new Device(); - gatewayDeviceTemplate.setName("mqtt_gateway " + RandomStringUtils.randomAlphabetic(5)); + gatewayDeviceTemplate.setName("mqtt_gateway_" + RandomStringUtils.randomAlphabetic(5)); gatewayDeviceTemplate.setType("gateway"); gatewayDeviceTemplate.setAdditionalInfo(additionalInfo); return gatewayDeviceTemplate; diff --git a/pom.xml b/pom.xml index 1c5dea9f5c..7d5540ef20 100755 --- a/pom.xml +++ b/pom.xml @@ -133,6 +133,10 @@ 1.3.0 1.2.7 + 7.6.1 + 3.23.1 + 5.2.0 + 1.3 1.17.3 1.12 3.0.0 @@ -1618,6 +1622,30 @@ + + org.testng + testng + ${testng.version} + test + + + org.assertj + assertj-core + ${assertj.version} + test + + + io.rest-assured + rest-assured + ${rest-assured.version} + test + + + org.hamcrest + hamcrest-all + ${hamcrest} + test + org.awaitility awaitility From 290e0894ff795352b835c79fa154787eb458e708 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 7 Nov 2022 15:04:32 +0200 Subject: [PATCH 11/80] added test listener for logging --- application/src/main/resources/logback.xml | 3 +- .../server/msa/AbstractContainerTest.java | 32 ++--------- .../thingsboard/server/msa/TestListener.java | 53 +++++++++++++++++++ .../msa/prototypes/DevicePrototypes.java | 4 +- pom.xml | 2 +- 5 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestListener.java diff --git a/application/src/main/resources/logback.xml b/application/src/main/resources/logback.xml index 90ed4785df..941bd7d278 100644 --- a/application/src/main/resources/logback.xml +++ b/application/src/main/resources/logback.xml @@ -25,8 +25,7 @@ - - + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index df1fe4a522..b85c8984bc 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -20,31 +20,21 @@ import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonObject; import com.google.gson.JsonParser; -import io.restassured.RestAssured; -import io.restassured.filter.log.RequestLoggingFilter; -import io.restassured.filter.log.ResponseLoggingFilter; import lombok.extern.slf4j.Slf4j; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.TrustStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; import org.testng.annotations.AfterSuite; import org.testng.annotations.BeforeSuite; +import org.testng.annotations.Listeners; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; - -import javax.net.ssl.SSLContext; import java.net.URI; import java.util.*; + @Slf4j +@Listeners(TestListener.class) public abstract class AbstractContainerTest { protected static long timeoutMultiplier = 1; @@ -161,20 +151,4 @@ public abstract class AbstractContainerTest { } } - public static HttpComponentsClientHttpRequestFactory getRequestFactoryForSelfSignedCert() throws Exception { - SSLContextBuilder builder = SSLContexts.custom(); - builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true); - SSLContext sslContext = builder.build(); - SSLConnectionSocketFactory sslSelfSigned = new SSLConnectionSocketFactory(sslContext, (s, sslSession) -> true); - - Registry socketFactoryRegistry = RegistryBuilder - .create() - .register("https", sslSelfSigned) - .build(); - - PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build(); - return new HttpComponentsClientHttpRequestFactory(httpClient); - } - } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestListener.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestListener.java new file mode 100644 index 0000000000..51bc75c86a --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestListener.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import lombok.extern.slf4j.Slf4j; +import org.testng.ITestContext; +import org.testng.ITestResult; +import org.testng.TestListenerAdapter; + +import static org.testng.internal.Utils.log; + +@Slf4j +public class TestListener extends TestListenerAdapter { + + @Override + public void onTestStart(ITestResult result) { + super.onTestStart(result); + log.info("===>>> Test started: " + result.getName()); + } + + /** + * Invoked when a test succeeds + */ + @Override + public void onTestSuccess(ITestResult result) { + super.onTestSuccess(result); + if (result != null) { + log.info("<<<=== Test completed successfully: " + result.getName()); + } + } + + /** + * Invoked when a test fails + */ + @Override + public void onTestFailure(ITestResult result) { + super.onTestFailure(result); + log.info("<<<=== Test failed: " + result.getName()); + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java index e8ed09df18..a9fc2f55e9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java @@ -30,11 +30,11 @@ public class DevicePrototypes { return device; } - public static Device defaultGatewayPrototype() throws JsonProcessingException { + public static Device defaultGatewayPrototype() { String isGateway = "{\"gateway\":true}"; JsonNode additionalInfo = JacksonUtil.valueToTree(isGateway); Device gatewayDeviceTemplate = new Device(); - gatewayDeviceTemplate.setName("mqtt_gateway_" + RandomStringUtils.randomAlphabetic(5)); + gatewayDeviceTemplate.setName("mqtt_gateway_" + RandomStringUtils.randomAlphanumeric(5)); gatewayDeviceTemplate.setType("gateway"); gatewayDeviceTemplate.setAdditionalInfo(additionalInfo); return gatewayDeviceTemplate; diff --git a/pom.xml b/pom.xml index 7d5540ef20..38718586f6 100755 --- a/pom.xml +++ b/pom.xml @@ -1643,7 +1643,7 @@ org.hamcrest hamcrest-all - ${hamcrest} + ${hamcrest.version} test From 610baa40bfb7f9b3ba5d040cfd9e182c40bc445e Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 7 Nov 2022 17:26:17 +0200 Subject: [PATCH 12/80] updated surefire plugin to run testNG tests --- msa/black-box-tests/README.md | 4 ++++ msa/black-box-tests/pom.xml | 13 ++++++++++--- .../msa/connectivity/MqttGatewayClientTest.java | 1 - msa/black-box-tests/src/test/resources/testNG.xml | 10 ++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 msa/black-box-tests/src/test/resources/testNG.xml diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index a45badf44a..a60e7405a8 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -30,5 +30,9 @@ As result, in REPOSITORY column, next images should be present: mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.hybridMode=true +To run the black box tests with using local env run tests in the [msa/black-box-tests](../black-box-tests) directory with runLocal property: + + mvn clean install -DblackBoxTests.skip=false -DrunLocal=true + diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index 9cd38c0c05..49f4c959e9 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -167,11 +167,18 @@ org.apache.maven.plugins maven-surefire-plugin - - **/*TestSuite.java - + + src/test/resources/testNG.xml + ${blackBoxTests.skip} + + + org.apache.maven.surefire + surefire-testng + ${surefire.version} + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index 54aec26421..ab42abc88e 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -27,7 +27,6 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; diff --git a/msa/black-box-tests/src/test/resources/testNG.xml b/msa/black-box-tests/src/test/resources/testNG.xml new file mode 100644 index 0000000000..c5ad55c9d1 --- /dev/null +++ b/msa/black-box-tests/src/test/resources/testNG.xml @@ -0,0 +1,10 @@ + + + + + + + + + + \ No newline at end of file From f9a4d87c39677ee23fe986890a17ace1731071a0 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 7 Nov 2022 18:01:46 +0200 Subject: [PATCH 13/80] refactoring --- .../server/msa/AbstractContainerTest.java | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index b85c8984bc..97d77ad9bf 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.ssl.SSLContextBuilder; @@ -37,10 +36,8 @@ import java.util.*; @Listeners(TestListener.class) public abstract class AbstractContainerTest { protected static long timeoutMultiplier = 1; - protected ObjectMapper mapper = new ObjectMapper(); - protected JsonParser jsonParser = new JsonParser(); - protected ContainerTestSuite containerTestSuite = ContainerTestSuite.getInstance(); + private static final ContainerTestSuite containerTestSuite = ContainerTestSuite.getInstance(); protected static TestRestClient testRestClient; @BeforeSuite @@ -95,27 +92,6 @@ public abstract class AbstractContainerTest { .build(); } - protected JsonObject createGatewayConnectPayload(String deviceName){ - JsonObject payload = new JsonObject(); - payload.addProperty("device", deviceName); - return payload; - } - - protected JsonObject createGatewayPayload(String deviceName, long ts){ - JsonObject payload = new JsonObject(); - payload.add(deviceName, createGatewayTelemetryArray(ts)); - return payload; - } - - protected JsonArray createGatewayTelemetryArray(long ts){ - JsonArray telemetryArray = new JsonArray(); - if (ts > 0) - telemetryArray.add(createPayload(ts)); - else - telemetryArray.add(createPayload()); - return telemetryArray; - } - protected JsonObject createPayload(long ts) { JsonObject values = createPayload(); JsonObject payload = new JsonObject(); From 30edf4da3c2fd2417b5c39664b7016b5409893a7 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 7 Nov 2022 18:25:41 +0200 Subject: [PATCH 14/80] refactoring --- .../server/msa/AbstractContainerTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index 97d77ad9bf..e9e6036771 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -19,6 +19,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonObject; +import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.ssl.SSLContextBuilder; @@ -37,6 +38,7 @@ import java.util.*; public abstract class AbstractContainerTest { protected static long timeoutMultiplier = 1; protected ObjectMapper mapper = new ObjectMapper(); + protected JsonParser jsonParser = new JsonParser(); private static final ContainerTestSuite containerTestSuite = ContainerTestSuite.getInstance(); protected static TestRestClient testRestClient; @@ -92,6 +94,27 @@ public abstract class AbstractContainerTest { .build(); } + protected JsonObject createGatewayConnectPayload(String deviceName){ + JsonObject payload = new JsonObject(); + payload.addProperty("device", deviceName); + return payload; + } + + protected JsonObject createGatewayPayload(String deviceName, long ts){ + JsonObject payload = new JsonObject(); + payload.add(deviceName, createGatewayTelemetryArray(ts)); + return payload; + } + + protected JsonArray createGatewayTelemetryArray(long ts){ + JsonArray telemetryArray = new JsonArray(); + if (ts > 0) + telemetryArray.add(createPayload(ts)); + else + telemetryArray.add(createPayload()); + return telemetryArray; + } + protected JsonObject createPayload(long ts) { JsonObject values = createPayload(); JsonObject payload = new JsonObject(); From 71af93eb4a8c92ef55e041abba8aa2e6efb7f1bd Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Mon, 7 Nov 2022 18:59:27 +0200 Subject: [PATCH 15/80] fixed test --- .../thingsboard/server/msa/prototypes/DevicePrototypes.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java index a9fc2f55e9..7db35460a9 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java @@ -15,9 +15,7 @@ */ package org.thingsboard.server.msa.prototypes; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.common.data.Device; @@ -32,7 +30,7 @@ public class DevicePrototypes { public static Device defaultGatewayPrototype() { String isGateway = "{\"gateway\":true}"; - JsonNode additionalInfo = JacksonUtil.valueToTree(isGateway); + JsonNode additionalInfo = JacksonUtil.toJsonNode(isGateway); Device gatewayDeviceTemplate = new Device(); gatewayDeviceTemplate.setName("mqtt_gateway_" + RandomStringUtils.randomAlphanumeric(5)); gatewayDeviceTemplate.setType("gateway"); From d9774ccfda1a58256dbab72bd06203be0d012e08 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 7 Sep 2022 16:44:05 +0300 Subject: [PATCH 16/80] http client returns headers as array is there is more than one, put headers to metadata for failure response --- .../rule/engine/rest/TbHttpClient.java | 22 ++++++++++++++++++- .../rule/engine/rest/TbHttpClientTest.java | 22 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 58bb5c7564..b7414c5e0d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -41,6 +41,7 @@ import org.springframework.util.concurrent.ListenableFutureCallback; import org.springframework.web.client.AsyncRestTemplate; import org.springframework.web.client.RestClientResponseException; import org.springframework.web.util.UriComponentsBuilder; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TbRelationTypes; @@ -59,8 +60,11 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.Deque; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; @Data @Slf4j @@ -244,17 +248,33 @@ public class TbHttpClient { metaData.putValue(STATUS, response.getStatusCode().name()); metaData.putValue(STATUS_CODE, response.getStatusCode().value() + ""); metaData.putValue(STATUS_REASON, response.getStatusCode().getReasonPhrase()); - response.getHeaders().toSingleValueMap().forEach(metaData::putValue); + headersToMetaData(response.getHeaders(), metaData::putValue); String body = response.getBody() == null ? "{}" : response.getBody(); return ctx.transformMsg(origMsg, origMsg.getType(), origMsg.getOriginator(), metaData, body); } + void headersToMetaData(Map> headers, BiConsumer consumer) { + if (headers == null) { + return; + } + headers.forEach((key, values) -> { + if (values != null && !values.isEmpty()) { + if (values.size() == 1) { + consumer.accept(key, values.get(0)); + } else { + consumer.accept(key, JacksonUtil.toString(values)); + } + } + }); + } + private TbMsg processFailureResponse(TbContext ctx, TbMsg origMsg, ResponseEntity response) { TbMsgMetaData metaData = origMsg.getMetaData(); metaData.putValue(STATUS, response.getStatusCode().name()); metaData.putValue(STATUS_CODE, response.getStatusCode().value() + ""); metaData.putValue(STATUS_REASON, response.getStatusCode().getReasonPhrase()); metaData.putValue(ERROR_BODY, response.getBody()); + headersToMetaData(response.getHeaders(), metaData::putValue); return ctx.transformMsg(origMsg, origMsg.getType(), origMsg.getOriginator(), metaData, origMsg.getData()); } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java index f7a1d83ac9..28918538e2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.rest; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; @@ -26,6 +27,7 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockserver.integration.ClientAndServer; +import org.springframework.util.LinkedMultiValueMap; import org.springframework.web.client.AsyncRestTemplate; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.id.DeviceId; @@ -34,6 +36,8 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.net.URI; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -200,5 +204,23 @@ public class TbHttpClientTest { ); } + @Test + public void testHeadersToMetaData() { + Map> headers = new LinkedMultiValueMap<>(); + headers.put("Content-Type", List.of("binary")); + headers.put("Set-Cookie", List.of("sap-context=sap-client=075; path=/", "sap-token=sap-client=075; path=/")); + + TbMsgMetaData metaData = new TbMsgMetaData(); + + willCallRealMethod().given(client).headersToMetaData(any(), any()); + + client.headersToMetaData(headers, metaData::putValue); + + Map data = metaData.getData(); + + Assertions.assertThat(data).hasSize(2); + Assertions.assertThat(data.get("Content-Type")).isEqualTo("binary"); + Assertions.assertThat(data.get("Set-Cookie")).isEqualTo("[\"sap-context=sap-client=075; path=/\",\"sap-token=sap-client=075; path=/\"]"); + } } \ No newline at end of file From 323d3d51389b53e77b88e8dfe63f03ad97e3d531 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Wed, 9 Nov 2022 16:33:43 +0200 Subject: [PATCH 17/80] fix bug for findAlarmDataByQueryForEntities method when sortOrder is null and textSearch used --- .../sql/query/DefaultAlarmQueryRepository.java | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 69cb4a2d86..66219eb9ba 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -140,6 +140,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { selectPart.append(" a.originator_id as entity_id "); } EntityDataSortOrder sortOrder = pageLink.getSortOrder(); + String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch()); if (sortOrder != null && sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { String sortOrderKey = sortOrder.getKey().getKey(); sortPart.append(alarmFieldColumnMap.getOrDefault(sortOrderKey, sortOrderKey)) @@ -166,7 +167,11 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } joinPart.append(" as e(id, priority)) e "); if (pageLink.isSearchPropagatedAlarms()) { - joinPart.append("on ea.entity_id = e.id"); + if (textSearchQuery.isEmpty()) { + joinPart.append("on ea.entity_id = e.id"); + } else { + joinPart.append("on a.entity_id = e.id"); + } } else { joinPart.append("on a.originator_id = e.id"); } @@ -230,13 +235,11 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } } - String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch()); - String mainQuery; - if (!textSearchQuery.isEmpty()) { - mainQuery = selectPart.toString() + fromPart.toString() + wherePart.toString(); - mainQuery = String.format("select * from (%s) a %s WHERE %s", mainQuery, joinPart, textSearchQuery); + String mainQuery = String.format("%s%s", selectPart, fromPart); + if (textSearchQuery.isEmpty()) { + mainQuery += String.format("%s%s", joinPart, wherePart); } else { - mainQuery = selectPart.toString() + fromPart.toString() + joinPart.toString() + wherePart.toString(); + mainQuery = String.format("select * from (%s%s) a %s WHERE %s", mainQuery, wherePart, joinPart, textSearchQuery); } String countQuery = String.format("select count(*) from (%s) result", mainQuery); long queryTs = System.currentTimeMillis(); From 7186632e5a37f6e63e14a144903c2ced6584c667 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 9 Nov 2022 23:24:09 +0100 Subject: [PATCH 18/80] lombok.copyableAnnotations += org.springframework.context.annotation.Lazy --- lombok.config | 1 + 1 file changed, 1 insertion(+) diff --git a/lombok.config b/lombok.config index d904701090..2299612bb9 100644 --- a/lombok.config +++ b/lombok.config @@ -1,2 +1,3 @@ config.stopbubbling = true lombok.anyconstructor.addconstructorproperties = true +lombok.copyableAnnotations += org.springframework.context.annotation.Lazy \ No newline at end of file From 1a9b8a1ebe27b232b3874af8e15f06b85c7ef64f Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Wed, 9 Nov 2022 23:28:34 +0100 Subject: [PATCH 19/80] JwtSettingsService workout: Lazy and Optional clusterService, correctness on first Install and upgrade, reload JWT on cluster notification, update jwt settings using existing id --- .../server/config/jwt/JwtSettingsService.java | 2 + .../config/jwt/JwtSettingsServiceDefault.java | 56 +++++++++++++------ .../server/controller/AdminController.java | 2 + .../queue/DefaultTbCoreConsumerService.java | 6 +- .../DefaultTbRuleEngineConsumerService.java | 3 +- .../processing/AbstractConsumerService.java | 22 +++++--- 6 files changed, 63 insertions(+), 28 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java index fb673fe50c..252b0a021c 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java @@ -19,6 +19,8 @@ public interface JwtSettingsService { JwtSettings getJwtSettings(); + void reloadJwtSettings(); + void createJwtAdminSettings(); JwtSettings saveJwtSettings(JwtSettings jwtSettings); diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java index 5c172c9c2d..d65edaa93b 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java @@ -19,7 +19,10 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.RandomStringUtils; -import org.springframework.dao.InvalidDataAccessResourceUsageException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; +import org.springframework.core.env.Profiles; import org.springframework.stereotype.Service; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.server.cluster.TbClusterService; @@ -33,6 +36,7 @@ import javax.validation.ValidationException; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Objects; +import java.util.Optional; @Service @RequiredArgsConstructor @@ -42,24 +46,37 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; static final String TB_ALLOW_DEFAULT_JWT_SIGNING_KEY = "TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"; - + @Lazy private final AdminSettingsService adminSettingsService; - private final TbClusterService tbClusterService; - + @Lazy + private final Optional tbClusterService; private final JwtSettingsValidator jwtSettingsValidator; - + private final Environment environment; @Getter private final JwtSettings jwtSettings; + @Value("${install.upgrade:false}") + private boolean isUpgrade; @PostConstruct public void init() { - reloadJwtSettings(); + if (!isFirstInstall()) { + reloadJwtSettings(); + } + } + + private boolean isInstall() { + return environment.acceptsProfiles(Profiles.of("install")); } - void reloadJwtSettings() { + private boolean isFirstInstall() { + return isInstall() && !isUpgrade; + } + + @Override + public void reloadJwtSettings() { AdminSettings adminJwtSettings = findJwtAdminSettings(); if (adminJwtSettings != null) { - log.debug("Loading the JWT admin settings from database"); + log.info("Reloading the JWT admin settings from database"); JwtSettings jwtLoaded = mapAdminToJwtSettings(adminJwtSettings); jwtSettings.setRefreshTokenExpTime(jwtLoaded.getRefreshTokenExpTime()); jwtSettings.setTokenExpirationTime(jwtLoaded.getTokenExpirationTime()); @@ -67,7 +84,7 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { jwtSettings.setTokenSigningKey(jwtLoaded.getTokenSigningKey()); } - if (hasDefaultTokenSigningKey()) { + if (hasDefaultTokenSigningKey() && !isFirstInstall()) { log.warn("JWT token signing key is default. This is a security issue. Please, consider to set unique value"); } } @@ -107,12 +124,20 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { } @Override - public JwtSettings saveJwtSettings(JwtSettings jwtSettings){ + public JwtSettings saveJwtSettings(JwtSettings jwtSettings) { jwtSettingsValidator.validate(jwtSettings); - AdminSettings adminJwtSettings = mapJwtToAdminSettings(jwtSettings); + final AdminSettings adminJwtSettings = mapJwtToAdminSettings(jwtSettings); + final AdminSettings existedSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + if (existedSettings != null) { + adminJwtSettings.setId(existedSettings.getId()); + } + log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); - tbClusterService.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, ComponentLifecycleEvent.UPDATED); + + if (!isInstall()) { + tbClusterService.orElseThrow().broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, ComponentLifecycleEvent.UPDATED); + } reloadJwtSettings(); return getJwtSettings(); } @@ -122,12 +147,7 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { } AdminSettings findJwtAdminSettings() { - try { - return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); - } catch (InvalidDataAccessResourceUsageException ignored) { - log.debug("findAdminSettingsByKey is returning InvalidDataAccessResourceUsageException. This is an installation case when the database is not initialized yet"); - return null; - } + return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); } /* diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 6b5977cac7..3ea4e5e13c 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -22,6 +22,7 @@ import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; @@ -68,6 +69,7 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Lazy @Autowired private JwtSettingsService jwtSettingsService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index ec58ac1aa9..d10c968364 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; @@ -143,8 +144,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService jwtSettingsService) { + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer(), jwtSettingsService); this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer(); this.firmwareStatesConsumer = tbCoreQueueFactory.createToOtaPackageStateServiceMsgConsumer(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index d870af318e..dd97db3703 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -70,6 +70,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -126,7 +127,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, QueueService queueService) { - super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); this.statisticsService = statisticsService; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.submitStrategyFactory = submitStrategyFactory; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 47b7f3f9f9..df7f0922ea 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -76,11 +77,13 @@ public abstract class AbstractConsumerService> nfConsumer; + protected final Optional jwtSettingsService; + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbAssetProfileCache assetProfileCache, TbApiUsageStateService apiUsageStateService, - PartitionService partitionService, TbQueueConsumer> nfConsumer) { + PartitionService partitionService, TbQueueConsumer> nfConsumer, Optional jwtSettingsService) { this.actorContext = actorContext; this.encodingService = encodingService; this.tenantProfileCache = tenantProfileCache; @@ -89,6 +92,7 @@ public abstract class AbstractConsumerService Date: Thu, 10 Nov 2022 10:41:34 +0100 Subject: [PATCH 20/80] saveJwtSettings returns JwtTokenPair in AdminController --- .../server/controller/AdminController.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 3ea4e5e13c..2712f58439 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -43,6 +43,9 @@ import org.thingsboard.server.config.jwt.JwtSettings; import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; @@ -73,6 +76,10 @@ public class AdminController extends BaseController { @Autowired private JwtSettingsService jwtSettingsService; + @Lazy + @Autowired + private JwtTokenFactory tokenFactory; + @Autowired private EntitiesVersionControlService versionControlService; @@ -175,19 +182,20 @@ public class AdminController extends BaseController { } } - @ApiOperation(value = "Update JWT Settings (saveSecuritySettings)", + @ApiOperation(value = "Update JWT Settings (saveJwtSettings)", notes = "Updates the JWT Settings object that contains JWT token policy, etc. The tokenSigningKey field is a Base64 encoded string." + SYSTEM_AUTHORITY_PARAGRAPH, produces = MediaType.APPLICATION_JSON_VALUE) @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/jwtSettings", method = RequestMethod.POST) @ResponseBody - public JwtSettings saveJwtSettings( + public JwtTokenPair saveJwtSettings( @ApiParam(value = "A JSON value representing the JWT Settings.") @RequestBody JwtSettings jwtSettings) throws ThingsboardException { try { - accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.WRITE); - jwtSettings = checkNotNull(jwtSettingsService.saveJwtSettings(jwtSettings)); - return jwtSettings; + SecurityUser securityUser = getCurrentUser(); + accessControlService.checkPermission(securityUser, Resource.ADMIN_SETTINGS, Operation.WRITE); + checkNotNull(jwtSettingsService.saveJwtSettings(jwtSettings)); + return tokenFactory.createTokenPair(securityUser); } catch (Exception e) { throw handleException(e); } From 1b191acace1f28bf6ab8b0643384cdb3041ff79f Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Thu, 10 Nov 2022 17:23:46 +0200 Subject: [PATCH 21/80] RestClient - add getRuleChains by type. Updated edge_root_rule_chain - remove Success from RPC Call from device to Push to cloud --- .../edge_management/rule_chains/edge_root_rule_chain.json | 5 ----- .../main/java/org/thingsboard/rest/client/RestClient.java | 7 ++++++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/application/src/main/data/json/tenant/edge_management/rule_chains/edge_root_rule_chain.json b/application/src/main/data/json/tenant/edge_management/rule_chains/edge_root_rule_chain.json index ae2e43fa73..717fc74715 100644 --- a/application/src/main/data/json/tenant/edge_management/rule_chains/edge_root_rule_chain.json +++ b/application/src/main/data/json/tenant/edge_management/rule_chains/edge_root_rule_chain.json @@ -174,11 +174,6 @@ "fromIndex": 3, "toIndex": 7, "type": "Timeseries Updated" - }, - { - "fromIndex": 4, - "toIndex": 7, - "type": "Success" } ], "ruleChainConnections": null diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 21a18534ed..fabc35bc49 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -2032,10 +2032,15 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { } public PageData getRuleChains(PageLink pageLink) { + return getRuleChains(RuleChainType.CORE, pageLink); + } + + public PageData getRuleChains(RuleChainType ruleChainType, PageLink pageLink) { Map params = new HashMap<>(); + params.put("type", ruleChainType.name()); addPageLinkToParam(params, pageLink); return restTemplate.exchange( - baseURL + "/api/ruleChains?" + getUrlParams(pageLink), + baseURL + "/api/ruleChains?type={type}&" + getUrlParams(pageLink), HttpMethod.GET, HttpEntity.EMPTY, new ParameterizedTypeReference>() { From 1ab4c03622544e61a00efc4d43a715cd7ae1fd23 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Thu, 10 Nov 2022 18:30:05 +0200 Subject: [PATCH 22/80] fixed license format --- .../src/test/resources/testNG.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/msa/black-box-tests/src/test/resources/testNG.xml b/msa/black-box-tests/src/test/resources/testNG.xml index c5ad55c9d1..45e93f76f1 100644 --- a/msa/black-box-tests/src/test/resources/testNG.xml +++ b/msa/black-box-tests/src/test/resources/testNG.xml @@ -1,4 +1,21 @@ + From b95f1c95e09c23a0c552c660f25521440233136c Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Nov 2022 18:00:54 +0100 Subject: [PATCH 23/80] jwt settings workout on review feedback --- .../config/jwt/JwtSettingsServiceDefault.java | 6 +++++- .../config/jwt/JwtSettingsValidator.java | 19 ++++++++++++++----- .../processing/AbstractConsumerService.java | 1 + 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java index d65edaa93b..f64b7c4658 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java @@ -164,7 +164,11 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { if (isAllowedDefaultJwtSigningKey()) { log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); } else { - String message = "Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. Key is a Base64 encoded phrase. This will require to generate new tokens for all users and API that uses JWT tokens. To allow insecure JWS use TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true"; + String message = "UPGRADE ERROR. YOUR ACTION REQUIRED. Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. " + + "The key should be a Base64 encoded string representing at least 256 bits of data. " + + "This will require to generate new tokens for all UI users and scripts that use JWT. " + + "To keep the default non-secure JWT signing key set TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true and restart the upgrade. " + + "You may change the JWT signing key later in the Admin Settings UI."; log.error(message); throw new ValidationException(message); } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java index 4e91c654e0..c6ec252d9f 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java @@ -16,6 +16,7 @@ package org.thingsboard.server.config.jwt; import lombok.AllArgsConstructor; +import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.Arrays; import org.springframework.stereotype.Component; @@ -23,6 +24,7 @@ import org.thingsboard.server.dao.exception.DataValidationException; import java.util.Base64; import java.util.Optional; +import java.util.concurrent.TimeUnit; @Component @AllArgsConstructor @@ -32,11 +34,14 @@ public class JwtSettingsValidator { if (StringUtils.isEmpty(jwtSettings.getTokenIssuer())) { throw new DataValidationException("JWT token issuer should be specified!"); } - if (Optional.ofNullable(jwtSettings.getRefreshTokenExpTime()).orElse(0) <= 0) { - throw new DataValidationException("JWT refresh token expiration time should be specified!"); + if (Optional.ofNullable(jwtSettings.getRefreshTokenExpTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(15)) { + throw new DataValidationException("JWT refresh token expiration time should be at least 15 minutes!"); } - if (Optional.ofNullable(jwtSettings.getTokenExpirationTime()).orElse(0) <= 0) { - throw new DataValidationException("JWT token expiration time should be specified!"); + if (Optional.ofNullable(jwtSettings.getTokenExpirationTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(1)) { + throw new DataValidationException("JWT token expiration time should be at least 1 minute!"); + } + if (jwtSettings.getTokenExpirationTime() >= jwtSettings.getRefreshTokenExpTime()) { + throw new DataValidationException("JWT token expiration time should greater than JWT refresh token expiration time!"); } if (StringUtils.isEmpty(jwtSettings.getTokenSigningKey())) { throw new DataValidationException("JWT token signing key should be specified!"); @@ -52,7 +57,11 @@ public class JwtSettingsValidator { if (Arrays.isNullOrEmpty(decodedKey)) { throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!"); } - Arrays.fill(decodedKey, (byte) 0); + if (decodedKey.length * Byte.SIZE < 256) { + throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 256 bits of data!"); + } + + System.arraycopy(decodedKey, 0, RandomUtils.nextBytes(decodedKey.length), 0, decodedKey.length); //secure memory } } diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index df7f0922ea..8d3ec4b55a 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -178,6 +178,7 @@ public abstract class AbstractConsumerService Date: Thu, 10 Nov 2022 19:06:21 +0100 Subject: [PATCH 24/80] jwt settings validation except install, warn on each reload if token key is default --- .../server/config/jwt/JwtSettingsService.java | 2 - .../config/jwt/JwtSettingsServiceDefault.java | 47 ++++--------- .../config/jwt/JwtSettingsValidator.java | 51 +------------- .../jwt/JwtSettingsValidatorDefault.java | 69 +++++++++++++++++++ .../jwt/JwtSettingsValidatorInstall.java} | 23 +++---- .../install/ThingsboardInstallService.java | 6 -- .../ConditionValidatorUpgradeService.java | 22 ------ 7 files changed, 94 insertions(+), 126 deletions(-) create mode 100644 application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java rename application/src/main/java/org/thingsboard/server/{service/install/ConditionValidatorUpgradeServiceImpl.java => config/jwt/JwtSettingsValidatorInstall.java} (58%) delete mode 100644 application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java index 252b0a021c..bb0127a39e 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java @@ -25,6 +25,4 @@ public interface JwtSettingsService { JwtSettings saveJwtSettings(JwtSettings jwtSettings); - void validateJwtTokenSigningKey(); - } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java index f64b7c4658..82657c5952 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java @@ -32,7 +32,6 @@ import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; import org.thingsboard.server.dao.settings.AdminSettingsService; import javax.annotation.PostConstruct; -import javax.validation.ValidationException; import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Objects; @@ -84,8 +83,10 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { jwtSettings.setTokenSigningKey(jwtLoaded.getTokenSigningKey()); } - if (hasDefaultTokenSigningKey() && !isFirstInstall()) { - log.warn("JWT token signing key is default. This is a security issue. Please, consider to set unique value"); + if (hasDefaultTokenSigningKey()) { + log.warn("WARNING: The platform is configured to use default JWT Signing Key. " + + "This is a security issue that needs to be resolved. Please change the JWT Signing Key using the Web UI. " + + "Navigate to \"System settings -> Security settings\" while logged in as a System Administrator."); } } @@ -107,17 +108,18 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { return TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey()); } + /** + * Create JWT admin settings is intended to be called from Install or Upgrade scripts + * */ @Override public void createJwtAdminSettings() { - log.debug("Creating JWT admin settings..."); + log.info("Creating JWT admin settings..."); Objects.requireNonNull(jwtSettings, "JWT settings is null"); if (isJwtAdminSettingsNotExists()) { - if (hasDefaultTokenSigningKey()) { - if (!isAllowedDefaultJwtSigningKey()) { - log.info("JWT token signing key is default. Generating a new random key"); - jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString( - RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); - } + if (hasDefaultTokenSigningKey() && isFirstInstall()) { + log.info("JWT token signing key is default. Generating a new random key"); + jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString( + RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); } saveJwtSettings(jwtSettings); } @@ -150,29 +152,4 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { return adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); } - /* - * Allowing default JWT signing key is not secure - * */ - boolean isAllowedDefaultJwtSigningKey() { - String allowDefaultJwtSigningKey = System.getenv(TB_ALLOW_DEFAULT_JWT_SIGNING_KEY); - return "true".equalsIgnoreCase(allowDefaultJwtSigningKey); - } - - @Override - public void validateJwtTokenSigningKey() { - if (isJwtAdminSettingsNotExists() && hasDefaultTokenSigningKey()) { - if (isAllowedDefaultJwtSigningKey()) { - log.warn("Default JWT signing key is allowed. This is a security issue. Please, consider to set a strong key in admin settings"); - } else { - String message = "UPGRADE ERROR. YOUR ACTION REQUIRED. Please, set a unique signing key with env variable JWT_TOKEN_SIGNING_KEY. " + - "The key should be a Base64 encoded string representing at least 256 bits of data. " + - "This will require to generate new tokens for all UI users and scripts that use JWT. " + - "To keep the default non-secure JWT signing key set TB_ALLOW_DEFAULT_JWT_SIGNING_KEY=true and restart the upgrade. " + - "You may change the JWT signing key later in the Admin Settings UI."; - log.error(message); - throw new ValidationException(message); - } - } - } - } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java index c6ec252d9f..8e0d4afe7a 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java @@ -15,53 +15,6 @@ */ package org.thingsboard.server.config.jwt; -import lombok.AllArgsConstructor; -import org.apache.commons.lang3.RandomUtils; -import org.apache.commons.lang3.StringUtils; -import org.bouncycastle.util.Arrays; -import org.springframework.stereotype.Component; -import org.thingsboard.server.dao.exception.DataValidationException; - -import java.util.Base64; -import java.util.Optional; -import java.util.concurrent.TimeUnit; - -@Component -@AllArgsConstructor -public class JwtSettingsValidator { - - public void validate(JwtSettings jwtSettings) { - if (StringUtils.isEmpty(jwtSettings.getTokenIssuer())) { - throw new DataValidationException("JWT token issuer should be specified!"); - } - if (Optional.ofNullable(jwtSettings.getRefreshTokenExpTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(15)) { - throw new DataValidationException("JWT refresh token expiration time should be at least 15 minutes!"); - } - if (Optional.ofNullable(jwtSettings.getTokenExpirationTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(1)) { - throw new DataValidationException("JWT token expiration time should be at least 1 minute!"); - } - if (jwtSettings.getTokenExpirationTime() >= jwtSettings.getRefreshTokenExpTime()) { - throw new DataValidationException("JWT token expiration time should greater than JWT refresh token expiration time!"); - } - if (StringUtils.isEmpty(jwtSettings.getTokenSigningKey())) { - throw new DataValidationException("JWT token signing key should be specified!"); - } - - byte[] decodedKey; - try { - decodedKey = Base64.getDecoder().decode(jwtSettings.getTokenSigningKey()); - } catch (Exception e) { - throw new DataValidationException("JWT token signing key should be valid Base64 encoded string! " + e.getCause()); - } - - if (Arrays.isNullOrEmpty(decodedKey)) { - throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!"); - } - if (decodedKey.length * Byte.SIZE < 256) { - throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 256 bits of data!"); - } - - System.arraycopy(decodedKey, 0, RandomUtils.nextBytes(decodedKey.length), 0, decodedKey.length); //secure memory - } - +public interface JwtSettingsValidator { + void validate(JwtSettings jwtSettings); } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java new file mode 100644 index 0000000000..ee60a80de6 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.config.jwt; + +import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.RandomUtils; +import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.Arrays; +import org.springframework.stereotype.Component; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.Base64; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +@Component +@RequiredArgsConstructor +public class JwtSettingsValidatorDefault implements JwtSettingsValidator { + + @Override + public void validate(JwtSettings jwtSettings) { + if (StringUtils.isEmpty(jwtSettings.getTokenIssuer())) { + throw new DataValidationException("JWT token issuer should be specified!"); + } + if (Optional.ofNullable(jwtSettings.getRefreshTokenExpTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(15)) { + throw new DataValidationException("JWT refresh token expiration time should be at least 15 minutes!"); + } + if (Optional.ofNullable(jwtSettings.getTokenExpirationTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(1)) { + throw new DataValidationException("JWT token expiration time should be at least 1 minute!"); + } + if (jwtSettings.getTokenExpirationTime() >= jwtSettings.getRefreshTokenExpTime()) { + throw new DataValidationException("JWT token expiration time should greater than JWT refresh token expiration time!"); + } + if (StringUtils.isEmpty(jwtSettings.getTokenSigningKey())) { + throw new DataValidationException("JWT token signing key should be specified!"); + } + + byte[] decodedKey; + try { + decodedKey = Base64.getDecoder().decode(jwtSettings.getTokenSigningKey()); + } catch (Exception e) { + throw new DataValidationException("JWT token signing key should be valid Base64 encoded string! " + e.getCause()); + } + + if (Arrays.isNullOrEmpty(decodedKey)) { + throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!"); + } + if (decodedKey.length * Byte.SIZE < 256) { + throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 256 bits of data!"); + } + + System.arraycopy(decodedKey, 0, RandomUtils.nextBytes(decodedKey.length), 0, decodedKey.length); //secure memory + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorInstall.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java rename to application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorInstall.java index 8dbfaab893..e353eb57a2 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeServiceImpl.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorInstall.java @@ -13,26 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.install; +package org.thingsboard.server.config.jwt; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Profile; -import org.springframework.stereotype.Service; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.springframework.stereotype.Component; -@Service +@Primary @Profile("install") +@Component @RequiredArgsConstructor -@Slf4j -public class ConditionValidatorUpgradeServiceImpl implements ConditionValidatorUpgradeService { - - private final JwtSettingsService jwtSettingsService; +public class JwtSettingsValidatorInstall implements JwtSettingsValidator { + /** + * During Install or upgrade the validation is suppressed to keep existing data + * */ @Override - public void validateConditionsBeforeUpgrade(String fromVersion) throws Exception { - log.info("Validating conditions before upgrade..."); - jwtSettingsService.validateJwtTokenSigningKey(); + public void validate(JwtSettings jwtSettings) { + } } diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index a100c61035..277e0b4d38 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -34,7 +34,6 @@ import org.thingsboard.server.service.install.migrate.EntitiesMigrateService; import org.thingsboard.server.service.install.migrate.TsLatestMigrateService; import org.thingsboard.server.service.install.update.CacheCleanupService; import org.thingsboard.server.service.install.update.DataUpdateService; -import org.thingsboard.server.service.install.ConditionValidatorUpgradeService; @Service @Profile("install") @@ -89,16 +88,11 @@ public class ThingsboardInstallService { @Autowired(required = false) private TsLatestMigrateService latestMigrateService; - @Autowired - private ConditionValidatorUpgradeService conditionValidatorUpgradeService; - public void performInstall() { try { if (isUpgrade) { log.info("Starting ThingsBoard Upgrade from version {} ...", upgradeFromVersion); - conditionValidatorUpgradeService.validateConditionsBeforeUpgrade(upgradeFromVersion); - cacheCleanupService.clearCache(upgradeFromVersion); if ("2.5.0-cassandra".equals(upgradeFromVersion)) { diff --git a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java b/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java deleted file mode 100644 index ec98a2f37e..0000000000 --- a/application/src/main/java/org/thingsboard/server/service/install/ConditionValidatorUpgradeService.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright © 2016-2022 The Thingsboard Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.thingsboard.server.service.install; - -public interface ConditionValidatorUpgradeService { - - void validateConditionsBeforeUpgrade(String fromVersion) throws Exception; - -} From b776cf13b60014b5363260a08614ac61117376c6 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Thu, 10 Nov 2022 23:48:08 +0100 Subject: [PATCH 25/80] jwt settings code cleanup --- .../server/config/jwt/JwtSettingsServiceDefault.java | 1 - application/src/main/resources/thingsboard.yml | 2 +- lombok.config | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java index 82657c5952..73867b1334 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java @@ -44,7 +44,6 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; - static final String TB_ALLOW_DEFAULT_JWT_SIGNING_KEY = "TB_ALLOW_DEFAULT_JWT_SIGNING_KEY"; @Lazy private final AdminSettingsService adminSettingsService; @Lazy diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 9d3eaf8f4a..ac6bed5b0f 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -107,7 +107,7 @@ plugins: # Security parameters security: # JWT Token parameters - jwt: # Since 3.5.0 values are persisted to the database during install or upgrade. On Install, the key will be generated randomly if no custom value set. + jwt: # Since 3.4.2 values are persisted to the database during install or upgrade. On Install, the key will be generated randomly if no custom value set. You can change it later from Web UI under SYS_ADMIN tokenExpirationTime: "${JWT_TOKEN_EXPIRATION_TIME:9000}" # Number of seconds (2.5 hours) refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:604800}" # Number of seconds (1 week). tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" diff --git a/lombok.config b/lombok.config index 2299612bb9..1b8f891cd9 100644 --- a/lombok.config +++ b/lombok.config @@ -1,3 +1,3 @@ config.stopbubbling = true lombok.anyconstructor.addconstructorproperties = true -lombok.copyableAnnotations += org.springframework.context.annotation.Lazy \ No newline at end of file +lombok.copyableAnnotations += org.springframework.context.annotation.Lazy From a00d68f477cd6390f2010c6d350b99425a778c5a Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 11 Nov 2022 10:06:02 +0200 Subject: [PATCH 26/80] delete cases: TENANT, DASHBOARD, CUSTOMER --- .../util/EntitiesByNameAndTypeLoader.java | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java index d70865f197..97bf05b2b9 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java @@ -45,15 +45,6 @@ public class EntitiesByNameAndTypeLoader { targetEntityId = asset.getId(); } break; - case CUSTOMER: - Optional customerOptional = ctx.getCustomerService().findCustomerByTenantIdAndTitle(ctx.getTenantId(), entityName); - if (customerOptional.isPresent()) { - targetEntityId = customerOptional.get().getId(); - } - break; - case TENANT: - targetEntityId = ctx.getTenantId(); - break; case ENTITY_VIEW: EntityView entityView = ctx.getEntityViewService().findEntityViewByTenantIdAndName(ctx.getTenantId(), entityName); if (entityView != null) { @@ -66,14 +57,8 @@ public class EntitiesByNameAndTypeLoader { targetEntityId = edge.getId(); } break; - case DASHBOARD: - DashboardInfo dashboardInfo = ctx.getDashboardService().findFirstDashboardInfoByTenantIdAndName(ctx.getTenantId(), entityName); - if (dashboardInfo != null) { - targetEntityId = dashboardInfo.getId(); - } - break; case USER: - User user = ctx.getUserService().findUserByEmail(ctx.getTenantId(), entityName); + User user = ctx.getUserService().findUserByTenantIdAndEmail(ctx.getTenantId(), entityName); if (user != null) { targetEntityId = user.getId(); } From b1d5e89e9c76d042a00f5a6c43e48931e288453c Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 11 Nov 2022 10:51:25 +0200 Subject: [PATCH 27/80] refactoring --- .../server/msa/AbstractContainerTest.java | 4 +- .../server/msa/ContainerTestSuite.java | 2 +- .../server/msa/TestProperties.java | 43 +++++++++++-------- .../server/msa/TestRestClient.java | 15 +++---- .../msa/connectivity/HttpClientTest.java | 4 +- .../msa/connectivity/MqttClientTest.java | 1 - .../connectivity/MqttGatewayClientTest.java | 12 ++++-- 7 files changed, 44 insertions(+), 37 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index e9e6036771..d647a6d927 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -29,8 +29,10 @@ import org.testng.annotations.BeforeSuite; import org.testng.annotations.Listeners; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.DeviceId; + import java.net.URI; -import java.util.*; +import java.util.Map; +import java.util.Random; @Slf4j diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index c32a6bdbe6..2ff7088fa0 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -54,7 +54,7 @@ public class ContainerTestSuite { private DockerComposeContainer testContainer; private ThingsBoardDbInstaller installTb; - public boolean isActive; + private boolean isActive; private static ContainerTestSuite containerTestSuite; diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java index 6e3a024cda..9acc2c2046 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -15,44 +15,49 @@ */ package org.thingsboard.server.msa; -import java.io.FileInputStream; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + import java.io.IOException; import java.io.InputStream; import java.util.Properties; +@Slf4j +@Component public class TestProperties { - public static final String HTTPS_URL = "https://localhost"; - protected static final String WSS_URL = "wss://localhost"; + private static final String HTTPS_URL = "https://localhost"; + + private static final String WSS_URL = "wss://localhost"; + + private static final ContainerTestSuite instance = ContainerTestSuite.getInstance(); - public static final ContainerTestSuite instance = ContainerTestSuite.getInstance(); + private static Properties properties; - public static String getBaseUrl(){ + public static String getBaseUrl() { if (instance.isActive()) { return HTTPS_URL; } - return getProperty("tb.baseUrl"); + return getProperties().getProperty("tb.baseUrl"); } - public static String getWebSocketUrl(){ + public static String getWebSocketUrl() { if (instance.isActive()) { return WSS_URL; } - return getProperty("tb.wsUrl"); + return getProperties().getProperty("tb.wsUrl"); } - private static String getProperty(String propertyName) { - - try (InputStream input = TestProperties.class.getClassLoader().getResourceAsStream("config.properties")) { - Properties prop = new Properties(); - prop.load(input); - return prop.getProperty(propertyName); - - } catch (IOException ex) { - ex.printStackTrace(); + private static Properties getProperties() { + if (properties == null) { + try (InputStream input = TestProperties.class.getClassLoader().getResourceAsStream("config.properties")) { + properties = new Properties(); + properties.load(input); + } catch (IOException ex) { + log.error("Exception while reading test properties " + ex.getMessage()); + } } - return null; - + return properties; } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index 0b0c4b89bc..b1907a0d99 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -17,10 +17,8 @@ package org.thingsboard.server.msa; import com.fasterxml.jackson.databind.JsonNode; import io.restassured.RestAssured; -import io.restassured.builder.ResponseSpecBuilder; import io.restassured.common.mapper.TypeRef; import io.restassured.config.HeaderConfig; -import io.restassured.config.HttpClientConfig; import io.restassured.config.RestAssuredConfig; import io.restassured.filter.log.RequestLoggingFilter; import io.restassured.filter.log.ResponseLoggingFilter; @@ -28,9 +26,10 @@ import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; -import io.restassured.specification.ResponseSpecification; -import org.hamcrest.Matchers; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -48,8 +47,6 @@ import java.util.List; import java.util.Map; import static io.restassured.RestAssured.given; -import static org.apache.http.params.CoreConnectionPNames.CONNECTION_TIMEOUT; -import static org.apache.http.params.CoreConnectionPNames.SO_TIMEOUT; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.AnyOf.anyOf; import static org.thingsboard.server.common.data.StringUtils.isEmpty; @@ -59,9 +56,7 @@ public class TestRestClient { private final String baseURL; private String token; private String refreshToken; - private RequestSpecification requestSpec; - private ResponseSpecification responseSpec; - protected static final String ACTIVATE_TOKEN_REGEX = "/api/noauth/activate?activateToken="; + private final RequestSpecification requestSpec; public TestRestClient(String url) { RestAssured.filters(new RequestLoggingFilter(), new ResponseLoggingFilter()); @@ -78,7 +73,7 @@ public class TestRestClient { } } - public void login(String username, String password) throws Exception { + public void login(String username, String password) { Map loginRequest = new HashMap<>(); loginRequest.put("username", username); loginRequest.put("password", password); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java index 6f24f00854..2eb4cdf076 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java @@ -25,12 +25,12 @@ import org.thingsboard.server.msa.AbstractContainerTest; import org.thingsboard.server.msa.WsClient; import org.thingsboard.server.msa.mapper.WsTelemetryResponse; - import java.util.Arrays; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; -import static org.thingsboard.server.common.data.DataConstants.*; +import static org.thingsboard.server.common.data.DataConstants.DEVICE; +import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; public class HttpClientTest extends AbstractContainerTest { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index 7e7ebf07ab..55dd432644 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -64,7 +64,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.testng.Assert.fail; import static org.thingsboard.server.common.data.DataConstants.DEVICE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; -import static org.thingsboard.server.msa.TestProperties.HTTPS_URL; import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; @Slf4j diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index ab42abc88e..c97214e18c 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -28,7 +28,6 @@ import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; @@ -52,8 +51,15 @@ import org.thingsboard.server.msa.mapper.WsTelemetryResponse; import java.io.IOException; import java.nio.charset.StandardCharsets; -import java.util.*; -import java.util.concurrent.*; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Random; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; import static org.thingsboard.server.common.data.DataConstants.DEVICE; From 62ca7447c44e8d239be4c76ff8fc3591bd503ffa Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 11 Nov 2022 11:06:36 +0200 Subject: [PATCH 28/80] refactoring --- .../server/msa/TestProperties.java | 2 - .../server/msa/TestRestClient.java | 40 +++++++++---------- 2 files changed, 19 insertions(+), 23 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java index 9acc2c2046..020dbf8b93 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -16,14 +16,12 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; import java.io.IOException; import java.io.InputStream; import java.util.Properties; @Slf4j -@Component public class TestProperties { private static final String HTTPS_URL = "https://localhost"; diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index b1907a0d99..a83913f842 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -26,10 +26,6 @@ import io.restassured.http.ContentType; import io.restassured.path.json.JsonPath; import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpStatus; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.TestPropertySource; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; @@ -47,6 +43,8 @@ import java.util.List; import java.util.Map; import static io.restassured.RestAssured.given; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.net.HttpURLConnection.HTTP_OK; import static org.hamcrest.Matchers.is; import static org.hamcrest.core.AnyOf.anyOf; import static org.thingsboard.server.common.data.StringUtils.isEmpty; @@ -54,9 +52,9 @@ import static org.thingsboard.server.common.data.StringUtils.isEmpty; public class TestRestClient { private static final String JWT_TOKEN_HEADER_PARAM = "X-Authorization"; private final String baseURL; + private final RequestSpecification requestSpec; private String token; private String refreshToken; - private final RequestSpecification requestSpec; public TestRestClient(String url) { RestAssured.filters(new RequestLoggingFilter(), new ResponseLoggingFilter()); @@ -90,7 +88,7 @@ public class TestRestClient { .pathParams("accessToken", accessToken) .post("/api/device?accessToken={accessToken}") .then() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(Device.class); } @@ -103,7 +101,7 @@ public class TestRestClient { .statusCode(statusCode); } public Device getDeviceById(DeviceId deviceId) { - return getDeviceById(deviceId, HttpStatus.OK.value()) + return getDeviceById(deviceId, HTTP_OK) .extract() .as(Device.class); } @@ -111,7 +109,7 @@ public class TestRestClient { return given().spec(requestSpec).get("/api/device/{deviceId}/credentials", deviceId.getId()) .then() .assertThat() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(DeviceCredentials.class); } @@ -120,34 +118,34 @@ public class TestRestClient { return given().spec(requestSpec).body(telemetry) .post("/api/v1/{credentialsId}/telemetry", credentialsId) .then() - .statusCode(HttpStatus.OK.value()); + .statusCode(HTTP_OK); } public ValidatableResponse deleteDevice(DeviceId deviceId) { return given().spec(requestSpec) .delete("/api/device/{deviceId}", deviceId.getId()) .then() - .statusCode(HttpStatus.OK.value()); + .statusCode(HTTP_OK); } public ValidatableResponse deleteDeviceIfExists(DeviceId deviceId) { return given().spec(requestSpec) .delete("/api/device/{deviceId}", deviceId.getId()) .then() - .statusCode(anyOf(is(HttpStatus.OK.value()),is(HttpStatus.NOT_FOUND.value()))); + .statusCode(anyOf(is(HTTP_OK),is(HTTP_NOT_FOUND))); } public ValidatableResponse postTelemetryAttribute(String entityType, DeviceId deviceId, String scope, JsonNode attribute) { return given().spec(requestSpec).body(attribute) .post("/api/plugins/telemetry/{entityType}/{entityId}/attributes/{scope}", entityType, deviceId.getId(), scope) .then() - .statusCode(HttpStatus.OK.value()); + .statusCode(HTTP_OK); } public ValidatableResponse postAttribute(String accessToken, JsonNode attribute) { return given().spec(requestSpec).body(attribute) .post("/api/v1/{accessToken}/attributes/", accessToken) .then() - .statusCode(HttpStatus.OK.value()); + .statusCode(HTTP_OK); } public JsonNode getAttributes(String accessToken, String clientKeys, String sharedKeys) { @@ -156,7 +154,7 @@ public class TestRestClient { .queryParam("sharedKeys", sharedKeys) .get("/api/v1/{accessToken}/attributes", accessToken) .then() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(JsonNode.class); } @@ -167,7 +165,7 @@ public class TestRestClient { return given().spec(requestSpec).queryParams(params) .get("/api/ruleChains") .then() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(new TypeRef>() {}); } @@ -177,7 +175,7 @@ public class TestRestClient { .body(ruleChain) .post("/api/ruleChain") .then() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(RuleChain.class); } @@ -187,7 +185,7 @@ public class TestRestClient { .body(ruleChainMetaData) .post("/api/ruleChain/metadata") .then() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(RuleChainMetaData.class); } @@ -196,14 +194,14 @@ public class TestRestClient { given().spec(requestSpec) .post("/api/ruleChain/{ruleChainId}/root", ruleChainId.getId()) .then() - .statusCode(HttpStatus.OK.value()); + .statusCode(HTTP_OK); } public void deleteRuleChain(RuleChainId ruleChainId) { given().spec(requestSpec) .delete("/api/ruleChain/{ruleChainId}", ruleChainId.getId()) .then() - .statusCode(HttpStatus.OK.value()); + .statusCode(HTTP_OK); } private String getUrlParams(PageLink pageLink) { @@ -239,7 +237,7 @@ public class TestRestClient { .pathParams(params) .get("/api/relations?fromId={fromId}&fromType={fromType}&relationTypeGroup={relationTypeGroup}") .then() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(new TypeRef>() {}); } @@ -249,7 +247,7 @@ public class TestRestClient { .body(serverRpcPayload) .post("/api/rpc/twoway/{deviceId}", deviceId.getId()) .then() - .statusCode(HttpStatus.OK.value()) + .statusCode(HTTP_OK) .extract() .as(JsonNode.class); } From e11b5ffedd381f94a11f8f1e02df2cf62ad4aed6 Mon Sep 17 00:00:00 2001 From: dashevchenko Date: Fri, 11 Nov 2022 12:33:16 +0200 Subject: [PATCH 29/80] refactoring --- .../thingsboard/server/msa/AbstractContainerTest.java | 2 -- .../org/thingsboard/server/msa/TestRestClient.java | 10 +++++----- .../server/msa/connectivity/MqttGatewayClientTest.java | 1 + 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index d647a6d927..a148d7d138 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -19,7 +19,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; import org.apache.http.conn.ssl.TrustStrategy; import org.apache.http.ssl.SSLContextBuilder; @@ -40,7 +39,6 @@ import java.util.Random; public abstract class AbstractContainerTest { protected static long timeoutMultiplier = 1; protected ObjectMapper mapper = new ObjectMapper(); - protected JsonParser jsonParser = new JsonParser(); private static final ContainerTestSuite containerTestSuite = ContainerTestSuite.getInstance(); protected static TestRestClient testRestClient; diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java index a83913f842..713205e228 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -51,7 +51,7 @@ import static org.thingsboard.server.common.data.StringUtils.isEmpty; public class TestRestClient { private static final String JWT_TOKEN_HEADER_PARAM = "X-Authorization"; - private final String baseURL; + private static final String CONTENT_TYPE_HEADER = "Content-Type"; private final RequestSpecification requestSpec; private String token; private String refreshToken; @@ -59,12 +59,11 @@ public class TestRestClient { public TestRestClient(String url) { RestAssured.filters(new RequestLoggingFilter(), new ResponseLoggingFilter()); - baseURL = url; - requestSpec = given().baseUri(baseURL) + requestSpec = given().baseUri(url) .contentType(ContentType.JSON) .config(RestAssuredConfig.config() .headerConfig(HeaderConfig.headerConfig() - .overwriteHeadersWithName(JWT_TOKEN_HEADER_PARAM))); + .overwriteHeadersWithName(JWT_TOKEN_HEADER_PARAM, CONTENT_TYPE_HEADER))); if (url.matches("^(https)://.*$")) { requestSpec.relaxedHTTPSValidation(); @@ -76,7 +75,8 @@ public class TestRestClient { loginRequest.put("username", username); loginRequest.put("password", password); - JsonPath jsonPath = given().relaxedHTTPSValidation().body(loginRequest).post(baseURL + "/api/auth/login") + JsonPath jsonPath = given().spec(requestSpec).body(loginRequest) + .post( "/api/auth/login") .getBody().jsonPath(); token = jsonPath.get("token"); refreshToken = jsonPath.get("refreshToken"); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index c97214e18c..84b2dd7253 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -72,6 +72,7 @@ public class MqttGatewayClientTest extends AbstractContainerTest { private MqttClient mqttClient; private Device createdDevice; private MqttMessageListener listener; + private JsonParser jsonParser = new JsonParser(); @BeforeMethod public void createGateway() throws Exception { From f49b34939677a98cb4e11916ea1592ef3c8d9f8d Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Fri, 11 Nov 2022 12:50:55 +0200 Subject: [PATCH 30/80] refactor code --- .../util/EntitiesByNameAndTypeLoader.java | 41 ++++--------------- 1 file changed, 9 insertions(+), 32 deletions(-) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java index 97bf05b2b9..3c3417ad25 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java @@ -16,60 +16,37 @@ package org.thingsboard.rule.engine.util; import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.DashboardInfo; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.id.EntityId; -import java.util.Optional; - public class EntitiesByNameAndTypeLoader { public static EntityId findEntityId(TbContext ctx, EntityType entityType, String entityName) { - EntityId targetEntityId = null; + SearchTextBasedWithAdditionalInfo targetEntity; switch (entityType) { case DEVICE: - Device device = ctx.getDeviceService().findDeviceByTenantIdAndName(ctx.getTenantId(), entityName); - if (device != null) { - targetEntityId = device.getId(); - } + targetEntity = ctx.getDeviceService().findDeviceByTenantIdAndName(ctx.getTenantId(), entityName); break; case ASSET: - Asset asset = ctx.getAssetService().findAssetByTenantIdAndName(ctx.getTenantId(), entityName); - if (asset != null) { - targetEntityId = asset.getId(); - } + targetEntity = ctx.getAssetService().findAssetByTenantIdAndName(ctx.getTenantId(), entityName); break; case ENTITY_VIEW: - EntityView entityView = ctx.getEntityViewService().findEntityViewByTenantIdAndName(ctx.getTenantId(), entityName); - if (entityView != null) { - targetEntityId = entityView.getId(); - } + targetEntity = ctx.getEntityViewService().findEntityViewByTenantIdAndName(ctx.getTenantId(), entityName); break; case EDGE: - Edge edge = ctx.getEdgeService().findEdgeByTenantIdAndName(ctx.getTenantId(), entityName); - if (edge != null) { - targetEntityId = edge.getId(); - } + targetEntity = ctx.getEdgeService().findEdgeByTenantIdAndName(ctx.getTenantId(), entityName); break; case USER: - User user = ctx.getUserService().findUserByTenantIdAndEmail(ctx.getTenantId(), entityName); - if (user != null) { - targetEntityId = user.getId(); - } + targetEntity = ctx.getUserService().findUserByTenantIdAndEmail(ctx.getTenantId(), entityName); break; default: throw new IllegalStateException("Unexpected entity type " + entityType.name()); } - if (targetEntityId == null) { + if (targetEntity == null) { throw new IllegalStateException("Failed to found " + entityType.name() + " entity by name: '" + entityName + "'!"); } - return targetEntityId; + return targetEntity.getId(); } } From 7b9c9e51cd165033a7654b1994daba3dc2bf2c04 Mon Sep 17 00:00:00 2001 From: ShvaykaD Date: Fri, 11 Nov 2022 14:41:15 +0200 Subject: [PATCH 31/80] update build of main query for alarm count query --- .../server/dao/sql/query/DefaultAlarmQueryRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 66219eb9ba..d91a8c5fc4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -237,7 +237,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { String mainQuery = String.format("%s%s", selectPart, fromPart); if (textSearchQuery.isEmpty()) { - mainQuery += String.format("%s%s", joinPart, wherePart); + mainQuery = String.format("%s%s%s", mainQuery, joinPart, wherePart); } else { mainQuery = String.format("select * from (%s%s) a %s WHERE %s", mainQuery, wherePart, joinPart, textSearchQuery); } From 8a29dc294e35f130d88438736f13c71a02f9d5b8 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Fri, 11 Nov 2022 17:41:12 +0200 Subject: [PATCH 32/80] mute TelemetryEdgeSqlTest that causes a lot of randomly generated errors --- application/src/test/resources/logback-test.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index d3301bf660..3762c8aa7c 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -16,6 +16,8 @@ + + From fc1da1299969dacf4f3d280f81905fd58ea751bd Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 14 Nov 2022 11:05:47 +0200 Subject: [PATCH 33/80] UI: Add JWT security settings form and restyle security page --- ui-ngx/src/app/core/http/admin.service.ts | 18 +- .../admin/security-settings.component.html | 373 +++++++++++------- .../admin/security-settings.component.scss | 23 +- .../admin/security-settings.component.ts | 132 ++++++- .../dialog/confirm-dialog.component.html | 2 +- .../src/app/shared/models/settings.models.ts | 7 + .../assets/locale/locale.constant-en_US.json | 20 + 7 files changed, 410 insertions(+), 165 deletions(-) diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 485f355b87..8933f7b99d 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -20,16 +20,18 @@ import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { AdminSettings, - RepositorySettings, + AutoCommitSettings, + JwtSettings, MailServerSettings, + RepositorySettings, + RepositorySettingsInfo, SecuritySettings, TestSmsRequest, - UpdateMessage, - AutoCommitSettings, - RepositorySettingsInfo + UpdateMessage } from '@shared/models/settings.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { tap } from 'rxjs/operators'; +import { LoginResponse } from '@shared/models/login.models'; @Injectable({ providedIn: 'root' @@ -70,6 +72,14 @@ export class AdminService { defaultHttpOptionsFromConfig(config)); } + public getJwtSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/jwtSettings`, defaultHttpOptionsFromConfig(config)); + } + + public saveJwtSettings(jwtSettings: JwtSettings, config?: RequestConfig): Observable { + return this.http.post('/api/admin/jwtSettings', jwtSettings, defaultHttpOptionsFromConfig(config)); + } + public getRepositorySettings(config?: RequestConfig): Observable { return this.http.get(`/api/admin/repositorySettings`, defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html index 80c9f384fd..7a498a0f20 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html @@ -15,144 +15,237 @@ limitations under the License. --> -
- - -
- admin.security-settings - -
-
- - - -
- -
-
-
- - - - -
admin.general-policy
- - - - admin.max-failed-login-attempts - - - {{ 'admin.minimum-max-failed-login-attempts-range' | translate }} - - - - admin.user-lockout-notification-email - - - - - - -
admin.password-policy
- - -
- - admin.minimum-password-length - - - {{ 'admin.minimum-password-length-required' | translate }} - - - {{ 'admin.minimum-password-length-range' | translate }} - - - {{ 'admin.minimum-password-length-range' | translate }} - - - - admin.minimum-uppercase-letters - - - {{ 'admin.minimum-uppercase-letters-range' | translate }} - - - - admin.minimum-lowercase-letters - - - {{ 'admin.minimum-lowercase-letters-range' | translate }} - - - - admin.minimum-digits - - - {{ 'admin.minimum-digits-range' | translate }} - - - - admin.minimum-special-characters - - - {{ 'admin.minimum-special-characters-range' | translate }} - - - - admin.password-expiration-period-days - - - {{ 'admin.password-expiration-period-days-range' | translate }} - - - - admin.password-reuse-frequency-days - - - {{ 'admin.password-reuse-frequency-days-range' | translate }} - - - - admin.allow-whitespace - -
- - -
-
- -
+ + +
+ admin.security-settings + +
+
+ + + +
+ + +
+
+ admin.general-policy + + admin.max-failed-login-attempts + + + {{ 'admin.minimum-max-failed-login-attempts-range' | translate }} + + + + admin.user-lockout-notification-email + + +
+ +
+ admin.password-policy +
+ + admin.minimum-password-length + + + {{ 'admin.minimum-password-length-required' | translate }} + + + {{ 'admin.minimum-password-length-range' | translate }} + + + {{ 'admin.minimum-password-length-range' | translate }} + + +
+ + admin.minimum-uppercase-letters + + + {{ 'admin.minimum-uppercase-letters-range' | translate }} + + + + admin.minimum-lowercase-letters + + + {{ 'admin.minimum-lowercase-letters-range' | translate }} + + +
+
+ + admin.minimum-digits + + + {{ 'admin.minimum-digits-range' | translate }} + + + + admin.minimum-special-characters + + + {{ 'admin.minimum-special-characters-range' | translate }} + + +
+
+ + admin.password-expiration-period-days + + + {{ 'admin.password-expiration-period-days-range' | translate }} + + + + admin.password-reuse-frequency-days + + + {{ 'admin.password-reuse-frequency-days-range' | translate }} + + +
+ + admin.allow-whitespace + +
- - - -
+
+ + +
+ + + + + + +
+ admin.jwt.security-settings +
+ + + +
+ +
+
+
+ + admin.jwt.issuer-name + + + {{ 'admin.jwt.issuer-name-required' | translate }} + + + + admin.jwt.signings-key + + + + {{ 'admin.jwt.signings-key-required' | translate }} + + + {{ 'admin.jwt.signings-key-base64' | translate }} + + +
+
+ + admin.jwt.expiration-time + + + {{ 'admin.jwt.expiration-time-required' | translate }} + + + {{ 'admin.jwt.expiration-time-pattern' | translate }} + + + {{ 'admin.jwt.expiration-time-min' | translate }} + + + + admin.jwt.refresh-expiration-time + + + {{ 'admin.jwt.refresh-expiration-time-required' | translate }} + + + {{ 'admin.jwt.refresh-expiration-time-pattern' | translate }} + + + {{ 'admin.jwt.refresh-expiration-time-min' | translate }} + + + {{ 'admin.jwt.refresh-expiration-time-less-token' | translate }} + + +
+
+ + +
+
+
+ + diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss index 32e514e010..5a6d5eed04 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss @@ -14,7 +14,26 @@ * limitations under the License. */ :host { - .mat-accordion-container { - margin-bottom: 16px; + .mat-headline { + margin-bottom: 8px; + } + + .mat-card-title { + margin: 0; + } + + .mat-card-content { + padding: 0 !important; + } + + .fields-group { + padding: 8px 16px 0; + margin: 10px 0; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + } } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts index 6dc070b278..bdf645a9d7 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts @@ -14,40 +14,50 @@ /// limitations under the License. /// -import { Component, OnInit } from '@angular/core'; +import { Component } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { PageComponent } from '@shared/components/page.component'; import { Router } from '@angular/router'; -import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { SecuritySettings } from '@shared/models/settings.models'; +import { FormBuilder, FormControl, FormGroup, Validators } from '@angular/forms'; +import { JwtSettings, SecuritySettings } from '@shared/models/settings.models'; import { AdminService } from '@core/http/admin.service'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { mergeMap, tap } from 'rxjs/operators'; +import { randomAlphanumeric } from '@core/utils'; +import { AuthService } from '@core/auth/auth.service'; +import { DialogService } from '@core/services/dialog.service'; +import { TranslateService } from '@ngx-translate/core'; +import { Observable, of } from 'rxjs'; @Component({ selector: 'tb-security-settings', templateUrl: './security-settings.component.html', styleUrls: ['./security-settings.component.scss', './settings-card.scss'] }) -export class SecuritySettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class SecuritySettingsComponent extends PageComponent implements HasConfirmForm { securitySettingsFormGroup: FormGroup; - securitySettings: SecuritySettings; + jwtSecuritySettingsFormGroup: FormGroup; + + private securitySettings: SecuritySettings; + private jwtSettings: JwtSettings; constructor(protected store: Store, private router: Router, private adminService: AdminService, - public fb: FormBuilder) { + private authService: AuthService, + private dialogService: DialogService, + private translate: TranslateService, + private fb: FormBuilder) { super(store); - } - - ngOnInit() { this.buildSecuritySettingsForm(); + this.buildJwtSecuritySettingsForm(); this.adminService.getSecuritySettings().subscribe( - (securitySettings) => { - this.securitySettings = securitySettings; - this.securitySettingsFormGroup.reset(this.securitySettings); - } + securitySettings => this.processSecuritySettings(securitySettings) + ); + this.adminService.getJwtSettings().subscribe( + jwtSettings => this.processJwtSettings(jwtSettings) ); } @@ -70,18 +80,104 @@ export class SecuritySettingsComponent extends PageComponent implements OnInit, }); } + buildJwtSecuritySettingsForm() { + this.jwtSecuritySettingsFormGroup = this.fb.group({ + tokenIssuer: ['', Validators.required], + tokenSigningKey: ['', [Validators.required, this.base64Format]], + tokenExpirationTime: [0, [Validators.required, Validators.pattern('[0-9]*'), Validators.min(60)]], + refreshTokenExpTime: [0, [Validators.required, Validators.pattern('[0-9]*'), Validators.min(900)]] + }, {validators: this.refreshTokenTimeGreatTokenTime.bind(this)}); + this.jwtSecuritySettingsFormGroup.get('tokenExpirationTime').valueChanges.subscribe( + () => this.jwtSecuritySettingsFormGroup.get('refreshTokenExpTime').updateValueAndValidity({onlySelf: true}) + ); + } + save(): void { this.securitySettings = {...this.securitySettings, ...this.securitySettingsFormGroup.value}; this.adminService.saveSecuritySettings(this.securitySettings).subscribe( - (securitySettings) => { - this.securitySettings = securitySettings; - this.securitySettingsFormGroup.reset(this.securitySettings); - } + securitySettings => this.processSecuritySettings(securitySettings) ); } + saveJwtSettings() { + const jwtFormSettings = this.jwtSecuritySettingsFormGroup.value; + this.confirmChangeJWTSettings().pipe(mergeMap(value => { + if (value) { + return this.adminService.saveJwtSettings(jwtFormSettings).pipe( + tap((data) => this.authService.setUserFromJwtToken(data.token, data.refreshToken, false)), + mergeMap(() => this.adminService.getJwtSettings()), + tap(jwtSettings => this.processJwtSettings(jwtSettings)) + ); + } + return of(null); + })).subscribe(() => {}); + } + + discardSetting() { + this.securitySettingsFormGroup.reset(this.securitySettings); + } + + discardJwtSetting() { + this.jwtSecuritySettingsFormGroup.reset(this.jwtSettings); + } + + private confirmChangeJWTSettings(): Observable { + if (this.jwtSecuritySettingsFormGroup.get('tokenIssuer').value !== (this.jwtSettings?.tokenIssuer || '') || + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').value !== (this.jwtSettings?.tokenSigningKey || '')) { + return this.dialogService.confirm( + this.translate.instant('admin.jwt.info-header'), + `
${this.translate.instant('admin.jwt.info-message')}
`, + this.translate.instant('action.discard-changes'), + this.translate.instant('action.confirm') + ); + } + return of(true); + } + + generateSigningKey() { + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').setValue(randomAlphanumeric(44)); + if (this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').pristine) { + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsDirty(); + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsTouched(); + } + } + + private processSecuritySettings(securitySettings: SecuritySettings) { + this.securitySettings = securitySettings; + this.securitySettingsFormGroup.reset(this.securitySettings); + } + + private processJwtSettings(jwtSettings: JwtSettings) { + this.jwtSettings = jwtSettings; + this.jwtSecuritySettingsFormGroup.reset(jwtSettings); + } + + private refreshTokenTimeGreatTokenTime(formGroup: FormGroup): { [key: string]: boolean } | null { + if (formGroup) { + const tokenTime = formGroup.value.tokenExpirationTime; + const refreshTokenTime = formGroup.value.refreshTokenExpTime; + if (tokenTime >= refreshTokenTime ) { + if (formGroup.get('refreshTokenExpTime').untouched) { + formGroup.get('refreshTokenExpTime').markAsTouched(); + } + formGroup.get('refreshTokenExpTime').setErrors({lessToken: true}); + return {lessToken: true}; + } + } + return null; + } + + private base64Format(control: FormControl): { [key: string]: boolean } | null { + try { + const value = btoa(control.value); + return null; + } catch (e) { + return {base64: true}; + } + } + confirmForm(): FormGroup { - return this.securitySettingsFormGroup; + return this.securitySettingsFormGroup.dirty ? this.securitySettingsFormGroup : this.jwtSecuritySettingsFormGroup; } } diff --git a/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html b/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html index 710339908b..f7586d5656 100644 --- a/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html +++ b/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html @@ -16,7 +16,7 @@ -->

{{data.title}}

-
+
diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 83426c8b3b..8ecb4066f1 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -63,6 +63,13 @@ export interface SecuritySettings { passwordPolicy: UserPasswordPolicy; } +export interface JwtSettings { + tokenIssuer: string; + tokenSigningKey: string; + tokenExpirationTime: number; + refreshTokenExpTime: number; +} + export interface UpdateMessage { message: string; updateAvailable: boolean; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3f3f70860f..9ad64b822e 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -381,6 +381,26 @@ "within-time": "Within time (sec)", "within-time-pattern": "Time must be a positive integer.", "within-time-required": "Time is required." + }, + "jwt": { + "security-settings": "JWT security settings", + "issuer-name": "Issuer name", + "issuer-name-required": "Issuer name is required.", + "signings-key": "Signing key", + "signings-key-required": "Signing key is required.", + "signings-key-base64": "Signing key must be base64 format.", + "expiration-time": "Token expiration time (sec)", + "expiration-time-required": "Token expiration time is required.", + "expiration-time-pattern": "Token expiration time be a positive integer.", + "expiration-time-min": "Minimum time is 60 seconds (1 minute).", + "refresh-expiration-time": "Refresh token expiration time", + "refresh-expiration-time-required": "Refresh token expiration time is required.", + "refresh-expiration-time-pattern": "Refresh token expiration time be a positive integer.", + "refresh-expiration-time-min": "Minimum time is 900 seconds (15 minute).", + "refresh-expiration-time-less-token": "Refresh token time must be greater token time.", + "generate-key": "Generate key", + "info-header": "All users will be to re-logined", + "info-message": "Change of the JWT Signing Key will cause all issued tokens to be invalid. All users will need to re-login. This will also affect scripts that use Rest API/Websockets." } }, "alarm": { From e32bd456b7875555f0b0b0654288f297e368fa0f Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 14 Nov 2022 11:07:09 +0100 Subject: [PATCH 34/80] added memory usage log to the js-executors --- msa/js-executor/api/jsInvokeMessageProcessor.ts | 5 +++++ msa/js-executor/config/custom-environment-variables.yml | 1 + msa/js-executor/config/default.yml | 1 + 3 files changed, 7 insertions(+) diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.ts b/msa/js-executor/api/jsInvokeMessageProcessor.ts index 668cd61f50..e69209fa01 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.ts +++ b/msa/js-executor/api/jsInvokeMessageProcessor.ts @@ -39,6 +39,7 @@ const TIMEOUT_ERROR = 2; const NOT_FOUND_ERROR = 3; const statFrequency = Number(config.get('script.stat_print_frequency')); +const memoryUsageTraceFrequency = Number(config.get('script.memory_usage_trace_frequency')); const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency')); const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); @@ -167,6 +168,10 @@ export class JsInvokeMessageProcessor { if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) { this.logger.info('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody); } + if (this.executedScriptsCounter % memoryUsageTraceFrequency == 0) { + this.logger.info('Current memory usage: [%s]', process.memoryUsage()); + } + this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then( (script) => { this.executor.executeScript(script, invokeRequest.args, invokeRequest.timeout).then( diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index b9c24c8d8d..2ebea4ccc1 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -75,6 +75,7 @@ logger: script: use_sandbox: "SCRIPT_USE_SANDBOX" + memory_usage_trace_frequency: "MEMORY_USAGE_TRACE_FREQUENCY" stat_print_frequency: "SCRIPT_STAT_PRINT_FREQUENCY" script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY" max_active_scripts: "MAX_ACTIVE_SCRIPTS" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 64829ef792..805c175dce 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -64,6 +64,7 @@ logger: script: use_sandbox: "true" + memory_usage_trace_frequency: "10000" script_body_trace_frequency: "10000" stat_print_frequency: "10000" max_active_scripts: "1000" From 11bd9b5257344b60c7e5eb1a2bbe6af175ffde5b Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Mon, 14 Nov 2022 11:28:30 +0100 Subject: [PATCH 35/80] added nodejs memory leak workaround --- docker/tb-js-executor.env | 3 ++- msa/js-executor/config/default.yml | 2 +- msa/js-executor/docker/start-js-executor.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index e080906549..1938449d53 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -3,4 +3,5 @@ LOGGER_LEVEL=info LOG_FOLDER=logs LOGGER_FILENAME=tb-js-executor-%DATE%.log DOCKER_MODE=true -SCRIPT_BODY_TRACE_FREQUENCY=1000 \ No newline at end of file +SCRIPT_BODY_TRACE_FREQUENCY=1000 +NODE_OPTIONS="--max-old-space-size=200" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 805c175dce..96f3401da5 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -64,7 +64,7 @@ logger: script: use_sandbox: "true" - memory_usage_trace_frequency: "10000" + memory_usage_trace_frequency: "1000" script_body_trace_frequency: "10000" stat_print_frequency: "10000" max_active_scripts: "1000" diff --git a/msa/js-executor/docker/start-js-executor.sh b/msa/js-executor/docker/start-js-executor.sh index 575f93c389..b27c1b7167 100755 --- a/msa/js-executor/docker/start-js-executor.sh +++ b/msa/js-executor/docker/start-js-executor.sh @@ -27,4 +27,4 @@ source "${CONF_FOLDER}/${configfile}" cd ${pkg.installFolder} # This will forward this PID 1 to the node.js and forward SIGTERM for graceful shutdown as well -exec node server.js +exec --no-compilation-cache node server.js From b02a2215db8f01b41a18049790d411ba5a3be647 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 14 Nov 2022 12:33:17 +0200 Subject: [PATCH 36/80] Update MVEL version to 2.4.24TB --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 60e8783bca..e7e75c32f8 100755 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 3.5.5 3.21.9 1.42.1 - 2.4.23TB + 2.4.24TB 1.18.18 1.2.4 4.1.75.Final From 335d88c82eaaf559bfd1b232419a7d6e714c4b67 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 14 Nov 2022 12:27:04 +0100 Subject: [PATCH 37/80] jwt settings added message on base64 validation exception --- .../server/config/jwt/JwtSettingsValidatorDefault.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java index ee60a80de6..66a7311bf3 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.config.jwt; -import lombok.AllArgsConstructor; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.StringUtils; @@ -53,7 +52,7 @@ public class JwtSettingsValidatorDefault implements JwtSettingsValidator { try { decodedKey = Base64.getDecoder().decode(jwtSettings.getTokenSigningKey()); } catch (Exception e) { - throw new DataValidationException("JWT token signing key should be valid Base64 encoded string! " + e.getCause()); + throw new DataValidationException("JWT token signing key should be a valid Base64 encoded string! " + e.getMessage()); } if (Arrays.isNullOrEmpty(decodedKey)) { From 5b1642246d722a67869ee4bb8ecca511cc138608 Mon Sep 17 00:00:00 2001 From: Yevhen Bondarenko <56396344+YevhenBondarenko@users.noreply.github.com> Date: Mon, 14 Nov 2022 13:30:05 +0100 Subject: [PATCH 38/80] [3.4.2] fix start js (#7614) * refactoring * fixed start js-executor typo --- msa/js-executor/docker/start-js-executor.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/msa/js-executor/docker/start-js-executor.sh b/msa/js-executor/docker/start-js-executor.sh index b27c1b7167..d30b62c145 100755 --- a/msa/js-executor/docker/start-js-executor.sh +++ b/msa/js-executor/docker/start-js-executor.sh @@ -27,4 +27,4 @@ source "${CONF_FOLDER}/${configfile}" cd ${pkg.installFolder} # This will forward this PID 1 to the node.js and forward SIGTERM for graceful shutdown as well -exec --no-compilation-cache node server.js +exec node --no-compilation-cache server.js From 51ec17d9d1782b755bd8aea10eeac1897c050166 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 14 Nov 2022 14:41:10 +0200 Subject: [PATCH 39/80] Update MVEL version to 2.4.25TB --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e7e75c32f8..5f5aa3703b 100755 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 3.5.5 3.21.9 1.42.1 - 2.4.24TB + 2.4.25TB 1.18.18 1.2.4 4.1.75.Final From 46b2adeb2cf4ead968df17a0d2e83e91b97f3b94 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 14 Nov 2022 13:43:56 +0100 Subject: [PATCH 40/80] jwt settings added message on base64 validation exception, default key is to be validated ok, tests added --- .../server/config/jwt/JwtSettings.java | 7 + .../config/jwt/JwtSettingsServiceDefault.java | 5 +- .../jwt/JwtSettingsValidatorDefault.java | 4 +- .../controller/BaseAdminControllerTest.java | 123 +++++++++++++----- 4 files changed, 101 insertions(+), 38 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java index f99b36f32e..2dd846446f 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java @@ -15,15 +15,22 @@ */ package org.thingsboard.server.config.jwt; +import lombok.AllArgsConstructor; import lombok.Data; +import lombok.NoArgsConstructor; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.security.model.JwtToken; @Component @ConfigurationProperties(prefix = "security.jwt") +@AllArgsConstructor +@NoArgsConstructor @Data public class JwtSettings { + static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; + static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; + /** * {@link JwtToken} will expire after this time. */ diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java index 73867b1334..6fac3b2ac2 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java @@ -37,13 +37,14 @@ import java.util.Base64; import java.util.Objects; import java.util.Optional; +import static org.thingsboard.server.config.jwt.JwtSettings.ADMIN_SETTINGS_JWT_KEY; +import static org.thingsboard.server.config.jwt.JwtSettings.TOKEN_SIGNING_KEY_DEFAULT; + @Service @RequiredArgsConstructor @Slf4j public class JwtSettingsServiceDefault implements JwtSettingsService { - static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; - static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; @Lazy private final AdminSettingsService adminSettingsService; @Lazy diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java index 66a7311bf3..840da625e6 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java +++ b/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java @@ -26,6 +26,8 @@ import java.util.Base64; import java.util.Optional; import java.util.concurrent.TimeUnit; +import static org.thingsboard.server.config.jwt.JwtSettings.TOKEN_SIGNING_KEY_DEFAULT; + @Component @RequiredArgsConstructor public class JwtSettingsValidatorDefault implements JwtSettingsValidator { @@ -58,7 +60,7 @@ public class JwtSettingsValidatorDefault implements JwtSettingsValidator { if (Arrays.isNullOrEmpty(decodedKey)) { throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!"); } - if (decodedKey.length * Byte.SIZE < 256) { + if (decodedKey.length * Byte.SIZE < 256 && !TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey())) { throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 256 bits of data!"); } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java index 81ecce8300..93269499d9 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java @@ -17,14 +17,22 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.Test; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.config.jwt.JwtSettings; +import org.thingsboard.server.config.jwt.JwtSettingsService; import org.thingsboard.server.service.mail.DefaultMailService; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -32,8 +40,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - +@Slf4j public abstract class BaseAdminControllerTest extends AbstractControllerTest { + final JwtSettings defaultJwtSettings = new JwtSettings(9000, "thingsboard.io", "thingsboardDefaultSigningKey", 604800); @Autowired MailService mailService; @@ -45,67 +54,67 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { public void testFindAdminSettingsByKey() throws Exception { loginSysAdmin(); doGet("/api/admin/settings/general") - .andExpect(status().isOk()) - .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$.id", notNullValue())) - .andExpect(jsonPath("$.key", is("general"))) - .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://localhost:8080"))); - + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.id", notNullValue())) + .andExpect(jsonPath("$.key", is("general"))) + .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://localhost:8080"))); + doGet("/api/admin/settings/mail") - .andExpect(status().isOk()) - .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$.id", notNullValue())) - .andExpect(jsonPath("$.key", is("mail"))) - .andExpect(jsonPath("$.jsonValue.smtpProtocol", is("smtp"))) - .andExpect(jsonPath("$.jsonValue.smtpHost", is("localhost"))) - .andExpect(jsonPath("$.jsonValue.smtpPort", is("25"))); - + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.id", notNullValue())) + .andExpect(jsonPath("$.key", is("mail"))) + .andExpect(jsonPath("$.jsonValue.smtpProtocol", is("smtp"))) + .andExpect(jsonPath("$.jsonValue.smtpHost", is("localhost"))) + .andExpect(jsonPath("$.jsonValue.smtpPort", is("25"))); + doGet("/api/admin/settings/unknown") - .andExpect(status().isNotFound()); - + .andExpect(status().isNotFound()); + } - + @Test public void testSaveAdminSettings() throws Exception { loginSysAdmin(); - AdminSettings adminSettings = doGet("/api/admin/settings/general", AdminSettings.class); - + AdminSettings adminSettings = doGet("/api/admin/settings/general", AdminSettings.class); + JsonNode jsonValue = adminSettings.getJsonValue(); ((ObjectNode) jsonValue).put("baseUrl", "http://myhost.org"); adminSettings.setJsonValue(jsonValue); doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); - + doGet("/api/admin/settings/general") - .andExpect(status().isOk()) - .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://myhost.org"))); - + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://myhost.org"))); + ((ObjectNode) jsonValue).put("baseUrl", "http://localhost:8080"); adminSettings.setJsonValue(jsonValue); - + doPost("/api/admin/settings", adminSettings) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } @Test public void testSaveAdminSettingsWithEmptyKey() throws Exception { loginSysAdmin(); - AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); + AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); adminSettings.setKey(null); doPost("/api/admin/settings", adminSettings) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Key should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Key should be specified"))); } - + @Test public void testChangeAdminSettingsKey() throws Exception { loginSysAdmin(); - AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); + AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); adminSettings.setKey("newKey"); doPost("/api/admin/settings", adminSettings) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("is prohibited"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("is prohibited"))); } @Test @@ -113,7 +122,7 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { loginSysAdmin(); AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); doPost("/api/admin/settings/testMail", adminSettings) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } @Test @@ -139,4 +148,48 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { doPost("/api/admin/settings/testMail", adminSettings).andExpect(status().is5xxServerError()); Mockito.doNothing().when(mailService).sendTestMail(Mockito.any(), Mockito.any()); } + + void resetJwtSettingsToDefault() throws Exception { + loginSysAdmin(); + doPost("/api/admin/jwtSettings", defaultJwtSettings).andExpect(status().isOk()); // jwt test scenarios are always started from + loginTenantAdmin(); + } + + @Test + public void testGetAndSaveDefaultJwtSettings() throws Exception { + JwtSettings jwtSettings; + loginSysAdmin(); + + jwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(defaultJwtSettings); + + doPost("/api/admin/jwtSettings", jwtSettings).andExpect(status().isOk()); + + jwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(defaultJwtSettings); + + resetJwtSettingsToDefault(); + } + + @Test + public void testCreateJwtSettings() throws Exception { + loginSysAdmin(); + + JwtSettings jwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(defaultJwtSettings); + + jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString( + RandomStringUtils.randomAlphanumeric(256 / Byte.SIZE).getBytes(StandardCharsets.UTF_8))); + + doPost("/api/admin/jwtSettings", jwtSettings).andExpect(status().isOk()); + + doGet("/api/admin/jwtSettings").andExpect(status().isUnauthorized()); //the old JWT token does not work after signing key was changed! + + loginSysAdmin(); + JwtSettings newJwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(newJwtSettings); + + resetJwtSettingsToDefault(); + } + } From 1d230f19451f1aa2178a818e0763330152b23af9 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 14 Nov 2022 15:03:55 +0200 Subject: [PATCH 41/80] UI: Fixed generate length signing key --- .../modules/home/pages/admin/security-settings.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts index bdf645a9d7..554d8550ff 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts @@ -126,7 +126,7 @@ export class SecuritySettingsComponent extends PageComponent implements HasConfi this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').value !== (this.jwtSettings?.tokenSigningKey || '')) { return this.dialogService.confirm( this.translate.instant('admin.jwt.info-header'), - `
${this.translate.instant('admin.jwt.info-message')}
`, + `
${this.translate.instant('admin.jwt.info-message')}
`, this.translate.instant('action.discard-changes'), this.translate.instant('action.confirm') ); @@ -135,7 +135,7 @@ export class SecuritySettingsComponent extends PageComponent implements HasConfi } generateSigningKey() { - this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').setValue(randomAlphanumeric(44)); + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').setValue(randomAlphanumeric(64)); if (this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').pristine) { this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsDirty(); this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsTouched(); From dde62fc51817f109d893713f8d7cd322ef45eaa6 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 14 Nov 2022 15:16:33 +0200 Subject: [PATCH 42/80] UI: Fixed generate length signing key --- .../modules/home/pages/admin/security-settings.component.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts index 554d8550ff..903bbb3bac 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts @@ -135,7 +135,7 @@ export class SecuritySettingsComponent extends PageComponent implements HasConfi } generateSigningKey() { - this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').setValue(randomAlphanumeric(64)); + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').setValue(btoa(randomAlphanumeric(64))); if (this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').pristine) { this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsDirty(); this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsTouched(); @@ -169,7 +169,7 @@ export class SecuritySettingsComponent extends PageComponent implements HasConfi private base64Format(control: FormControl): { [key: string]: boolean } | null { try { - const value = btoa(control.value); + const value = atob(control.value); return null; } catch (e) { return {base64: true}; From 25f8ff2aefed66c17adb558546cf7bc575c31257 Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Mon, 14 Nov 2022 15:37:54 +0200 Subject: [PATCH 43/80] Handle undefined script execution result. --- msa/js-executor/api/jsExecutor.models.ts | 2 +- msa/js-executor/api/jsInvokeMessageProcessor.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/msa/js-executor/api/jsExecutor.models.ts b/msa/js-executor/api/jsExecutor.models.ts index 7a6b53cd8a..17407f4d50 100644 --- a/msa/js-executor/api/jsExecutor.models.ts +++ b/msa/js-executor/api/jsExecutor.models.ts @@ -56,7 +56,7 @@ export interface JsCompileResponse extends TbMessage { export interface JsInvokeResponse { success: boolean; - result: string; + result?: string; errorCode?: number; errorDetails?: string; } diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.ts b/msa/js-executor/api/jsInvokeMessageProcessor.ts index e69209fa01..52a337c74a 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.ts +++ b/msa/js-executor/api/jsInvokeMessageProcessor.ts @@ -175,8 +175,8 @@ export class JsInvokeMessageProcessor { this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then( (script) => { this.executor.executeScript(script, invokeRequest.args, invokeRequest.timeout).then( - (result) => { - if (result.length <= maxResultSize) { + (result: string | undefined) => { + if (!result || result.length <= maxResultSize) { const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse(result, true); this.logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId); this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse); @@ -328,7 +328,7 @@ export class JsInvokeMessageProcessor { } } - private static createInvokeResponse(result: string, success: boolean, errorCode?: number, err?: any): JsInvokeResponse { + private static createInvokeResponse(result: string | undefined, success: boolean, errorCode?: number, err?: any): JsInvokeResponse { return { errorCode: errorCode, success: success, From dee81fea2653ab22aa9ae97c7b9dd33c1cb0b60a Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 14 Nov 2022 18:00:19 +0200 Subject: [PATCH 44/80] MVEL scripts caching --- .../src/main/resources/thingsboard.yml | 1 + .../service/script/MvelInvokeServiceTest.java | 98 +++++++++++++++++++ common/script/script-api/pom.xml | 4 + .../api/mvel/DefaultMvelInvokeService.java | 65 ++++++++++-- .../script/api/mvel/MvelScript.java | 1 - 5 files changed, 158 insertions(+), 11 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 0f110da90c..cd05c7ff23 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -625,6 +625,7 @@ mvel: max_black_list_duration_sec: "${MVEL_MAX_BLACKLIST_DURATION_SEC:60}" # Specify thread pool size for javascript executor service thread_pool_size: "${MVEL_THREAD_POOL_SIZE:50}" + compiled_scripts_cache_size: "${MVEL_COMPILED_SCRIPTS_CACHE_SIZE:2000}" stats: enabled: "${TB_MVEL_STATS_ENABLED:false}" print_interval_ms: "${TB_MVEL_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java index a40a087514..1c3c134a82 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.script; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.benmanes.caffeine.cache.Cache; import org.junit.Assert; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -24,15 +25,22 @@ import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.mvel.MvelInvokeService; +import org.thingsboard.script.api.mvel.MvelScript; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest @@ -41,6 +49,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; "mvel.max_total_args_size=50", "mvel.max_result_size=50", "mvel.max_errors=2", + "mvel.compiled_scripts_cache_size=100" }) class MvelInvokeServiceTest extends AbstractControllerTest { @@ -110,6 +119,89 @@ class MvelInvokeServiceTest extends AbstractControllerTest { assertThatScriptIsBlocked(scriptId); } + @Test + void givenScriptsWithSameBody_thenCompileAndCacheOnlyOnce() throws Exception { + String script = "return msg.temperature > 20;"; + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + } + + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Map scriptMap = getFieldValue(invokeService, "scriptMap"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + String scriptHash = scriptIdToHash.get(scriptsIds.get(0)); + + assertThat(scriptsIds.stream().map(scriptIdToHash::get)).containsOnly(scriptHash); + assertThat(scriptMap).containsKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNotNull(); + } + + @Test + public void whenReleasingScript_thenCheckForScriptHashUsages() throws Exception { + String script = "return msg.temperature > 20;"; + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + } + + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Map scriptMap = getFieldValue(invokeService, "scriptMap"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + String scriptHash = scriptIdToHash.get(scriptsIds.get(0)); + for (int i = 0; i < 9; i++) { + UUID scriptId = scriptsIds.get(i); + assertThat(scriptIdToHash).containsKey(scriptId); + invokeService.release(scriptId); + assertThat(scriptIdToHash).doesNotContainKey(scriptId); + } + assertThat(scriptMap).containsKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNotNull(); + + invokeService.release(scriptsIds.get(9)); + assertThat(scriptMap).doesNotContainKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNull(); + } + + @Test + public void whenCompiledScriptsCacheIsTooBig_thenRemoveRarelyUsedScripts() throws Exception { + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 110; i++) { // mvel.compiled_scripts_cache_size = 100 + String script = "return msg.temperature > " + i; + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + + for (int j = 0; j < i; j++) { + invokeScript(scriptId, "{ \"temperature\": 12 }"); // so that scriptsIds is ordered by number of invocations + } + } + + ConcurrentMap cache = compiledScriptsCache.asMap(); + + for (int i = 0; i < 10; i++) { // iterating rarely used scripts + UUID scriptId = scriptsIds.get(i); + String scriptHash = scriptIdToHash.get(scriptId); + assertThat(cache).doesNotContainKey(scriptHash); + } + for (int i = 10; i < 110; i++) { + UUID scriptId = scriptsIds.get(i); + String scriptHash = scriptIdToHash.get(scriptId); + assertThat(cache).containsKey(scriptHash); + } + + UUID scriptRemovedFromCache = scriptsIds.get(0); + assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNull(); + invokeScript(scriptRemovedFromCache, "{ \"temperature\": 12 }"); + assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNotNull(); + } + private void assertThatScriptIsBlocked(UUID scriptId) { assertThatThrownBy(() -> { invokeScript(scriptId, "{}"); @@ -125,4 +217,10 @@ class MvelInvokeServiceTest extends AbstractControllerTest { return invokeService.invokeScript(TenantId.SYS_TENANT_ID, null, scriptId, msg, "{}", "POST_TELEMETRY_REQUEST").get().toString(); } + private T getFieldValue(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(target); + } + } diff --git a/common/script/script-api/pom.xml b/common/script/script-api/pom.xml index 62dc00ca97..b7ae4993f9 100644 --- a/common/script/script-api/pom.xml +++ b/common/script/script-api/pom.xml @@ -56,6 +56,10 @@ com.google.code.gson gson + + com.github.ben-manes.caffeine + caffeine + org.slf4j slf4j-api diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java index 901a49180e..120be2af4f 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java @@ -15,6 +15,10 @@ */ package org.thingsboard.script.api.mvel; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -42,6 +46,7 @@ import org.thingsboard.server.common.stats.TbApiUsageStateClient; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; import java.util.Optional; @@ -55,7 +60,10 @@ import java.util.regex.Pattern; @Service public class DefaultMvelInvokeService extends AbstractScriptInvokeService implements MvelInvokeService { - protected Map scriptMap = new ConcurrentHashMap<>(); + protected final Map scriptIdToHash = new ConcurrentHashMap<>(); + protected final Map scriptMap = new ConcurrentHashMap<>(); + protected Cache compiledScriptsCache; + private SandboxedParserConfiguration parserConfig; private static final Pattern NEW_KEYWORD_PATTERN = Pattern.compile("new\\s"); @@ -92,6 +100,9 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Value("${mvel.max_memory_limit_mb:8}") private long maxMemoryLimitMb; + @Value("${mvel.compiled_scripts_cache_size:2000}") + private int compiledScriptsCacheSize; + private ListeningExecutorService executor; protected DefaultMvelInvokeService(Optional apiUsageStateClient, Optional apiUsageReportClient) { @@ -115,11 +126,14 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "mvel-executor")); try { // Special command to warm up MVEL engine - Serializable script = MVEL.compileExpression("var warmUp = {}; warmUp", new SandboxedParserContext(parserConfig)); + Serializable script = compileScript("var warmUp = {}; warmUp"); MVEL.executeTbExpression(script, new ExecutionContext(parserConfig), Collections.emptyMap()); } catch (Exception e) { // do nothing } + compiledScriptsCache = Caffeine.newBuilder() + .maximumSize(compiledScriptsCacheSize) + .build(); } @PreDestroy @@ -141,16 +155,21 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Override protected boolean isScriptPresent(UUID scriptId) { - return scriptMap.containsKey(scriptId); + return scriptIdToHash.containsKey(scriptId); } @Override protected ListenableFuture doEvalScript(TenantId tenantId, ScriptType scriptType, String scriptBody, UUID scriptId, String[] argNames) { return executor.submit(() -> { try { - Serializable compiledScript = MVEL.compileExpression(scriptBody, new SandboxedParserContext(parserConfig)); - MvelScript script = new MvelScript(compiledScript, scriptBody, argNames); - scriptMap.put(scriptId, script); + String scriptHash = hash(scriptBody, argNames); + compiledScriptsCache.get(scriptHash, k -> { + return compileScript(scriptBody); + }); + scriptIdToHash.put(scriptId, scriptHash); + scriptMap.computeIfAbsent(scriptHash, k -> { + return new MvelScript(scriptBody, argNames); + }); return scriptId; } catch (Exception e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.COMPILATION, scriptBody, e); @@ -162,12 +181,16 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem protected MvelScriptExecutionTask doInvokeFunction(UUID scriptId, Object[] args) { ExecutionContext executionContext = new ExecutionContext(this.parserConfig, maxMemoryLimitMb * 1024 * 1024); return new MvelScriptExecutionTask(executionContext, executor.submit(() -> { - MvelScript script = scriptMap.get(scriptId); - if (script == null) { + String scriptHash = scriptIdToHash.get(scriptId); + if (scriptHash == null) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.OTHER, null, new RuntimeException("Script not found!")); } + MvelScript script = scriptMap.get(scriptHash); + Serializable compiledScript = compiledScriptsCache.get(scriptHash, k -> { + return compileScript(script.getScriptBody()); + }); try { - return MVEL.executeTbExpression(script.getCompiledScript(), executionContext, script.createVars(args)); + return MVEL.executeTbExpression(compiledScript, executionContext, script.createVars(args)); } catch (ScriptMemoryOverflowException e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.OTHER, script.getScriptBody(), new RuntimeException("Script memory overflow!")); } catch (Exception e) { @@ -178,6 +201,28 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Override protected void doRelease(UUID scriptId) throws Exception { - scriptMap.remove(scriptId); + String scriptHash = scriptIdToHash.remove(scriptId); + if (scriptHash != null) { + if (scriptIdToHash.containsValue(scriptHash)) { + return; + } + scriptMap.remove(scriptHash); + compiledScriptsCache.invalidate(scriptHash); + } + } + + private Serializable compileScript(String scriptBody) { + return MVEL.compileExpression(scriptBody, new SandboxedParserContext(parserConfig)); } + + @SuppressWarnings("UnstableApiUsage") + protected String hash(String scriptBody, String[] argNames) { + Hasher hasher = Hashing.murmur3_128().newHasher(); + hasher.putUnencodedChars(scriptBody); + for (String argName : argNames) { + hasher.putString(argName, StandardCharsets.UTF_8); + } + return hasher.hash().toString(); + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java index 7a84c7b0af..bca5d8d546 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java @@ -24,7 +24,6 @@ import java.util.Map; @Data public class MvelScript { - private final Serializable compiledScript; private final String scriptBody; private final String[] argNames; From 6e12a168c036ec9e348b8460196806b99ac5ded4 Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Mon, 14 Nov 2022 18:57:52 +0200 Subject: [PATCH 45/80] UI: Fixed signing key validators --- .../home/pages/admin/security-settings.component.html | 9 +++++---- .../home/pages/admin/security-settings.component.ts | 7 +++++++ ui-ngx/src/assets/locale/locale.constant-en_US.json | 4 +++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html index 7a498a0f20..86f6f9b8c7 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html @@ -165,9 +165,6 @@ admin.jwt.security-settings
- - -
@@ -181,7 +178,7 @@ admin.jwt.signings-key - + + admin.jwt.signings-key-hint {{ 'admin.jwt.signings-key-required' | translate }} {{ 'admin.jwt.signings-key-base64' | translate }} + + {{ 'admin.jwt.signings-key-min-length' | translate }} +
diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts index 903bbb3bac..dc7e9c05b7 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts @@ -121,6 +121,10 @@ export class SecuritySettingsComponent extends PageComponent implements HasConfi this.jwtSecuritySettingsFormGroup.reset(this.jwtSettings); } + markAsTouched() { + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsTouched(); + } + private confirmChangeJWTSettings(): Observable { if (this.jwtSecuritySettingsFormGroup.get('tokenIssuer').value !== (this.jwtSettings?.tokenIssuer || '') || this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').value !== (this.jwtSettings?.tokenSigningKey || '')) { @@ -170,6 +174,9 @@ export class SecuritySettingsComponent extends PageComponent implements HasConfi private base64Format(control: FormControl): { [key: string]: boolean } | null { try { const value = atob(control.value); + if (value.length < 32 && control.value !== 'thingsboardDefaultSigningKey') { + return {minLength: true}; + } return null; } catch (e) { return {base64: true}; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 9ad64b822e..511d9def5d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -387,13 +387,15 @@ "issuer-name": "Issuer name", "issuer-name-required": "Issuer name is required.", "signings-key": "Signing key", + "signings-key-hint": "Base64 encoded string representing at least 256 bits of data.", "signings-key-required": "Signing key is required.", + "signings-key-min-length": "Signing key must be at least 256 bits of data.", "signings-key-base64": "Signing key must be base64 format.", "expiration-time": "Token expiration time (sec)", "expiration-time-required": "Token expiration time is required.", "expiration-time-pattern": "Token expiration time be a positive integer.", "expiration-time-min": "Minimum time is 60 seconds (1 minute).", - "refresh-expiration-time": "Refresh token expiration time", + "refresh-expiration-time": "Refresh token expiration time (sec)", "refresh-expiration-time-required": "Refresh token expiration time is required.", "refresh-expiration-time-pattern": "Refresh token expiration time be a positive integer.", "refresh-expiration-time-min": "Minimum time is 900 seconds (15 minute).", From 12830936815a5fed8c141fb771a44265378fe94f Mon Sep 17 00:00:00 2001 From: ViacheslavKlimov Date: Mon, 14 Nov 2022 19:02:52 +0200 Subject: [PATCH 46/80] Synchronize doEvalScript and doRelease operations in MvelInvokeService --- .../src/main/resources/thingsboard.yml | 2 +- .../api/mvel/DefaultMvelInvokeService.java | 31 +++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index cd05c7ff23..28149246c9 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -625,7 +625,7 @@ mvel: max_black_list_duration_sec: "${MVEL_MAX_BLACKLIST_DURATION_SEC:60}" # Specify thread pool size for javascript executor service thread_pool_size: "${MVEL_THREAD_POOL_SIZE:50}" - compiled_scripts_cache_size: "${MVEL_COMPILED_SCRIPTS_CACHE_SIZE:2000}" + compiled_scripts_cache_size: "${MVEL_COMPILED_SCRIPTS_CACHE_SIZE:1000}" stats: enabled: "${TB_MVEL_STATS_ENABLED:false}" print_interval_ms: "${TB_MVEL_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java index 120be2af4f..4fb029096b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java @@ -53,6 +53,8 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; @Slf4j @@ -100,11 +102,13 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Value("${mvel.max_memory_limit_mb:8}") private long maxMemoryLimitMb; - @Value("${mvel.compiled_scripts_cache_size:2000}") + @Value("${mvel.compiled_scripts_cache_size:1000}") private int compiledScriptsCacheSize; private ListeningExecutorService executor; + private final Lock lock = new ReentrantLock(); + protected DefaultMvelInvokeService(Optional apiUsageStateClient, Optional apiUsageReportClient) { super(apiUsageStateClient, apiUsageReportClient); } @@ -166,10 +170,15 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem compiledScriptsCache.get(scriptHash, k -> { return compileScript(scriptBody); }); - scriptIdToHash.put(scriptId, scriptHash); - scriptMap.computeIfAbsent(scriptHash, k -> { - return new MvelScript(scriptBody, argNames); - }); + lock.lock(); + try { + scriptIdToHash.put(scriptId, scriptHash); + scriptMap.computeIfAbsent(scriptHash, k -> { + return new MvelScript(scriptBody, argNames); + }); + } finally { + lock.unlock(); + } return scriptId; } catch (Exception e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.COMPILATION, scriptBody, e); @@ -203,11 +212,15 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem protected void doRelease(UUID scriptId) throws Exception { String scriptHash = scriptIdToHash.remove(scriptId); if (scriptHash != null) { - if (scriptIdToHash.containsValue(scriptHash)) { - return; + lock.lock(); + try { + if (!scriptIdToHash.containsValue(scriptHash)) { + scriptMap.remove(scriptHash); + compiledScriptsCache.invalidate(scriptHash); + } + } finally { + lock.unlock(); } - scriptMap.remove(scriptHash); - compiledScriptsCache.invalidate(scriptHash); } } From 7e5ab3bff70e133bb4477db346a9103f203ebfc5 Mon Sep 17 00:00:00 2001 From: Sergey Matvienko Date: Mon, 14 Nov 2022 18:52:57 +0100 Subject: [PATCH 47/80] JWT settings refactored packages, Swagger documented, rest client getJwtSettings and saveJwtSettings added --- .../server/ThingsboardInstallApplication.java | 2 +- .../server/controller/AdminController.java | 8 +++--- .../server/controller/AuthController.java | 6 ++--- .../controller/TwoFactorAuthController.java | 6 ++--- .../server/controller/UserController.java | 4 +-- .../DefaultSystemDataLoaderService.java | 2 +- .../queue/DefaultTbCoreConsumerService.java | 2 +- .../processing/AbstractConsumerService.java | 2 +- .../jwt/settings}/JwtSettingsService.java | 4 ++- .../settings}/JwtSettingsServiceDefault.java | 24 +++++++++++++---- .../jwt/settings}/JwtSettingsValidator.java | 7 ++++- .../JwtSettingsValidatorDefault.java | 5 ++-- .../JwtSettingsValidatorInstall.java | 3 ++- .../Oauth2AuthenticationSuccessHandler.java | 4 +-- ...RestAwareAuthenticationSuccessHandler.java | 4 +-- .../security/model/token/JwtTokenFactory.java | 8 +++--- .../controller/BaseAdminControllerTest.java | 5 ++-- .../server/controller/TwoFactorAuthTest.java | 4 +-- .../security/auth/JwtTokenFactoryTest.java | 4 +-- .../common/data/security/model/JwtPair.java | 9 +++---- .../data/security/model}/JwtSettings.java | 26 +++++++++---------- .../thingsboard/rest/client/RestClient.java | 19 ++++++++++++++ 22 files changed, 98 insertions(+), 60 deletions(-) rename application/src/main/java/org/thingsboard/server/{config/jwt => service/security/auth/jwt/settings}/JwtSettingsService.java (85%) rename application/src/main/java/org/thingsboard/server/{config/jwt => service/security/auth/jwt/settings}/JwtSettingsServiceDefault.java (84%) rename application/src/main/java/org/thingsboard/server/{config/jwt => service/security/auth/jwt/settings}/JwtSettingsValidator.java (73%) rename application/src/main/java/org/thingsboard/server/{config/jwt => service/security/auth/jwt/settings}/JwtSettingsValidatorDefault.java (95%) rename application/src/main/java/org/thingsboard/server/{config/jwt => service/security/auth/jwt/settings}/JwtSettingsValidatorInstall.java (89%) rename application/src/main/java/org/thingsboard/server/service/security/model/JwtTokenPair.java => common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtPair.java (85%) rename {application/src/main/java/org/thingsboard/server/config/jwt => common/data/src/main/java/org/thingsboard/server/common/data/security/model}/JwtSettings.java (65%) diff --git a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java index 6009dafafa..b4a0e019b6 100644 --- a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java +++ b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java @@ -29,10 +29,10 @@ import java.util.Arrays; @ComponentScan({"org.thingsboard.server.install", "org.thingsboard.server.service.component", "org.thingsboard.server.service.install", + "org.thingsboard.server.service.security.auth.jwt.settings", "org.thingsboard.server.dao", "org.thingsboard.server.common.stats", "org.thingsboard.server.common.transport.config.ssl", - "org.thingsboard.server.config.jwt", "org.thingsboard.server.cache", "org.thingsboard.server.springfox" }) diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 2712f58439..29f32b8ac9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -39,11 +39,11 @@ import org.thingsboard.server.common.data.sms.config.TestSmsRequest; import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettingsInfo; -import org.thingsboard.server.config.jwt.JwtSettings; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.thingsboard.server.common.data.security.model.JwtSettings; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; @@ -188,7 +188,7 @@ public class AdminController extends BaseController { @PreAuthorize("hasAuthority('SYS_ADMIN')") @RequestMapping(value = "/jwtSettings", method = RequestMethod.POST) @ResponseBody - public JwtTokenPair saveJwtSettings( + public JwtPair saveJwtSettings( @ApiParam(value = "A JSON value representing the JWT Settings.") @RequestBody JwtSettings jwtSettings) throws ThingsboardException { try { diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 8384599c4d..0cb3a3fc92 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -51,7 +51,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.service.security.model.ActivateUserRequest; import org.thingsboard.server.service.security.model.ChangePasswordRequest; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.ResetPasswordEmailRequest; import org.thingsboard.server.service.security.model.ResetPasswordRequest; import org.thingsboard.server.service.security.model.SecurityUser; @@ -236,7 +236,7 @@ public class AuthController extends BaseController { @RequestMapping(value = "/noauth/activate", method = RequestMethod.POST) @ResponseStatus(value = HttpStatus.OK) @ResponseBody - public JwtTokenPair activateUser( + public JwtPair activateUser( @ApiParam(value = "Activate user request.") @RequestBody ActivateUserRequest activateRequest, @RequestParam(required = false, defaultValue = "true") boolean sendActivationMail, @@ -278,7 +278,7 @@ public class AuthController extends BaseController { @RequestMapping(value = "/noauth/resetPassword", method = RequestMethod.POST) @ResponseStatus(value = HttpStatus.OK) @ResponseBody - public JwtTokenPair resetPassword( + public JwtPair resetPassword( @ApiParam(value = "Reset password request.") @RequestBody ResetPasswordRequest resetPasswordRequest, HttpServletRequest request) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java index 003b4ab450..5ce46e324e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java @@ -39,7 +39,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.system.SystemSecurityService; @@ -87,8 +87,8 @@ public class TwoFactorAuthController extends BaseController { "and Too Many Requests error if rate limits are exceeded.") @PostMapping("/verification/check") @PreAuthorize("hasAuthority('PRE_VERIFICATION_TOKEN')") - public JwtTokenPair checkTwoFaVerificationCode(@RequestParam TwoFaProviderType providerType, - @RequestParam String verificationCode, HttpServletRequest servletRequest) throws Exception { + public JwtPair checkTwoFaVerificationCode(@RequestParam TwoFaProviderType providerType, + @RequestParam String verificationCode, HttpServletRequest servletRequest) throws Exception { SecurityUser user = getCurrentUser(); boolean verificationSuccess = twoFactorAuthService.checkVerificationCode(user, providerType, verificationCode, true); if (verificationSuccess) { diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index cea9d2e95a..a2b3a6d6bc 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -46,7 +46,7 @@ import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.user.TbUserService; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -145,7 +145,7 @@ public class UserController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @RequestMapping(value = "/user/{userId}/token", method = RequestMethod.GET) @ResponseBody - public JwtTokenPair getUserToken( + public JwtPair getUserToken( @ApiParam(value = USER_ID_PARAM_DESCRIPTION) @PathVariable(USER_ID) String strUserId) throws ThingsboardException { checkParameter(USER_ID, strUserId); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 961d0265f9..e512fb028e 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -82,7 +82,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.data.widget.WidgetsBundle; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index d10c968364..d0af49b1e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -35,7 +35,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 8d3ec4b55a..c814ab1704 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -33,7 +33,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java similarity index 85% rename from application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java rename to application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java index bb0127a39e..0282e64804 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsService.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java @@ -13,7 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config.jwt; +package org.thingsboard.server.service.security.auth.jwt.settings; + +import org.thingsboard.server.common.data.security.model.JwtSettings; public interface JwtSettingsService { diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsServiceDefault.java similarity index 84% rename from application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java rename to application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsServiceDefault.java index 6fac3b2ac2..d5e92554f9 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsServiceDefault.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsServiceDefault.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config.jwt; +package org.thingsboard.server.service.security.auth.jwt.settings; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -29,6 +29,7 @@ import org.thingsboard.server.cluster.TbClusterService; import org.thingsboard.server.common.data.AdminSettings; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; +import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.dao.settings.AdminSettingsService; import javax.annotation.PostConstruct; @@ -37,8 +38,8 @@ import java.util.Base64; import java.util.Objects; import java.util.Optional; -import static org.thingsboard.server.config.jwt.JwtSettings.ADMIN_SETTINGS_JWT_KEY; -import static org.thingsboard.server.config.jwt.JwtSettings.TOKEN_SIGNING_KEY_DEFAULT; +import static org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsValidator.ADMIN_SETTINGS_JWT_KEY; +import static org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsValidator.TOKEN_SIGNING_KEY_DEFAULT; @Service @RequiredArgsConstructor @@ -52,12 +53,25 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { private final JwtSettingsValidator jwtSettingsValidator; private final Environment environment; @Getter - private final JwtSettings jwtSettings; + private final JwtSettings jwtSettings = new JwtSettings(); @Value("${install.upgrade:false}") private boolean isUpgrade; + @Value("${security.jwt.tokenExpirationTime:9000}") + private Integer tokenExpirationTime; + @Value("${security.jwt.refreshTokenExpTime:604800}") + private Integer refreshTokenExpTime; + @Value("${security.jwt.tokenIssuer:thingsboard.io}") + private String tokenIssuer; + @Value("${security.jwt.tokenSigningKey:thingsboardDefaultSigningKey}") + private String tokenSigningKey; + @PostConstruct public void init() { + jwtSettings.setTokenExpirationTime(this.tokenExpirationTime); + jwtSettings.setRefreshTokenExpTime(this.refreshTokenExpTime); + jwtSettings.setTokenIssuer(this.tokenIssuer); + jwtSettings.setTokenSigningKey(this.tokenSigningKey); if (!isFirstInstall()) { reloadJwtSettings(); } @@ -77,8 +91,8 @@ public class JwtSettingsServiceDefault implements JwtSettingsService { if (adminJwtSettings != null) { log.info("Reloading the JWT admin settings from database"); JwtSettings jwtLoaded = mapAdminToJwtSettings(adminJwtSettings); - jwtSettings.setRefreshTokenExpTime(jwtLoaded.getRefreshTokenExpTime()); jwtSettings.setTokenExpirationTime(jwtLoaded.getTokenExpirationTime()); + jwtSettings.setRefreshTokenExpTime(jwtLoaded.getRefreshTokenExpTime()); jwtSettings.setTokenIssuer(jwtLoaded.getTokenIssuer()); jwtSettings.setTokenSigningKey(jwtLoaded.getTokenSigningKey()); } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidator.java similarity index 73% rename from application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java rename to application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidator.java index 8e0d4afe7a..30d23bca00 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidator.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidator.java @@ -13,8 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config.jwt; +package org.thingsboard.server.service.security.auth.jwt.settings; + +import org.thingsboard.server.common.data.security.model.JwtSettings; public interface JwtSettingsValidator { + String ADMIN_SETTINGS_JWT_KEY = "jwt"; + String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; + void validate(JwtSettings jwtSettings); } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidatorDefault.java similarity index 95% rename from application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java rename to application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidatorDefault.java index 840da625e6..2ebe4e8025 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorDefault.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidatorDefault.java @@ -13,21 +13,20 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config.jwt; +package org.thingsboard.server.service.security.auth.jwt.settings; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.StringUtils; import org.bouncycastle.util.Arrays; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.dao.exception.DataValidationException; import java.util.Base64; import java.util.Optional; import java.util.concurrent.TimeUnit; -import static org.thingsboard.server.config.jwt.JwtSettings.TOKEN_SIGNING_KEY_DEFAULT; - @Component @RequiredArgsConstructor public class JwtSettingsValidatorDefault implements JwtSettingsValidator { diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorInstall.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidatorInstall.java similarity index 89% rename from application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorInstall.java rename to application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidatorInstall.java index e353eb57a2..a7d097ee39 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettingsValidatorInstall.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidatorInstall.java @@ -13,12 +13,13 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config.jwt; +package org.thingsboard.server.service.security.auth.jwt.settings; import lombok.RequiredArgsConstructor; import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.security.model.JwtSettings; @Primary @Profile("install") diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java index 9fd2a680b4..9be9d2217b 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.oauth2.OAuth2Registration; import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -104,7 +104,7 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(request, token, oAuth2AuthorizedClient.getAccessToken().getTokenValue(), registration); - JwtTokenPair tokenPair = tokenFactory.createTokenPair(securityUser); + JwtPair tokenPair = tokenFactory.createTokenPair(securityUser); clearAuthenticationAttributes(request, response); getRedirectStrategy().sendRedirect(request, response, baseUrl + "/?accessToken=" + tokenPair.getToken() + "&refreshToken=" + tokenPair.getRefreshToken()); diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index 4d7ef01914..f6d9fd8666 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -26,7 +26,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.auth.MfaAuthenticationToken; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -49,7 +49,7 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException { SecurityUser securityUser = (SecurityUser) authentication.getPrincipal(); - JwtTokenPair tokenPair = new JwtTokenPair(); + JwtPair tokenPair = new JwtPair(); if (authentication instanceof MfaAuthenticationToken) { int preVerificationTokenLifetime = twoFaConfigManager.getPlatformTwoFaSettings(securityUser.getTenantId(), true) diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index b5779dcb44..6da68c37f3 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -35,9 +35,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -214,10 +214,10 @@ public class JwtTokenFactory { } } - public JwtTokenPair createTokenPair(SecurityUser securityUser) { + public JwtPair createTokenPair(SecurityUser securityUser) { JwtToken accessToken = createAccessJwtToken(securityUser); JwtToken refreshToken = createRefreshToken(securityUser); - return new JwtTokenPair(accessToken.getToken(), refreshToken.getToken()); + return new JwtPair(accessToken.getToken(), refreshToken.getToken()); } } diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java index 93269499d9..285d647a14 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java @@ -25,8 +25,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; -import org.thingsboard.server.config.jwt.JwtSettings; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.service.mail.DefaultMailService; import java.nio.charset.StandardCharsets; @@ -42,7 +41,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. @Slf4j public abstract class BaseAdminControllerTest extends AbstractControllerTest { - final JwtSettings defaultJwtSettings = new JwtSettings(9000, "thingsboard.io", "thingsboardDefaultSigningKey", 604800); + final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "thingsboardDefaultSigningKey"); @Autowired MailService mailService; diff --git a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java index 9839843b0c..6c7dfc3cd7 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java @@ -51,7 +51,7 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.security.auth.rest.LoginRequest; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import java.time.Duration; import java.util.Arrays; @@ -396,7 +396,7 @@ public abstract class TwoFactorAuthTest extends AbstractControllerTest { private void logInWithPreVerificationToken(String username, String password) throws Exception { LoginRequest loginRequest = new LoginRequest(username, password); - JwtTokenPair response = readResponse(doPost("/api/auth/login", loginRequest).andExpect(status().isOk()), JwtTokenPair.class); + JwtPair response = readResponse(doPost("/api/auth/login", loginRequest).andExpect(status().isOk()), JwtPair.class); assertThat(response.getToken()).isNotNull(); assertThat(response.getRefreshToken()).isNull(); assertThat(response.getScope()).isEqualTo(Authority.PRE_VERIFICATION_TOKEN); diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java index 796ef93c0b..bf89eadda5 100644 --- a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java @@ -23,8 +23,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.jwt.JwtSettings; -import org.thingsboard.server.config.jwt.JwtSettingsService; +import org.thingsboard.server.common.data.security.model.JwtSettings; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.AccessJwtToken; diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/JwtTokenPair.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtPair.java similarity index 85% rename from application/src/main/java/org/thingsboard/server/service/security/model/JwtTokenPair.java rename to common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtPair.java index 02e28cd885..eb50a11a92 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/JwtTokenPair.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtPair.java @@ -13,19 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.security.model; +package org.thingsboard.server.common.data.security.model; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.security.Authority; -@ApiModel(value = "JWT Token Pair") +@ApiModel(value = "JWT Pair") @Data @NoArgsConstructor -public class JwtTokenPair { +public class JwtPair { @ApiModelProperty(position = 1, value = "The JWT Access Token. Used to perform API calls.", example = "AAB254FF67D..") private String token; @@ -34,7 +33,7 @@ public class JwtTokenPair { private Authority scope; - public JwtTokenPair(String token, String refreshToken) { + public JwtPair(String token, String refreshToken) { this.token = token; this.refreshToken = refreshToken; } diff --git a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java similarity index 65% rename from application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java index 2dd846446f..f5668ff088 100644 --- a/application/src/main/java/org/thingsboard/server/config/jwt/JwtSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java @@ -13,43 +13,43 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config.jwt; +package org.thingsboard.server.common.data.security.model; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.security.model.JwtToken; -@Component -@ConfigurationProperties(prefix = "security.jwt") +@ApiModel(value = "JWT Settings") @AllArgsConstructor @NoArgsConstructor @Data public class JwtSettings { - static final String ADMIN_SETTINGS_JWT_KEY = "jwt"; - static final String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; /** * {@link JwtToken} will expire after this time. */ + @ApiModelProperty(position = 1, value = "The JWT will expire after seconds.", example = "9000") private Integer tokenExpirationTime; + /** + * {@link JwtToken} can be refreshed during this timeframe. + */ + @ApiModelProperty(position = 2, value = "The JWT can be refreshed during seconds.", example = "604800") + private Integer refreshTokenExpTime; + /** * Token issuer. */ + @ApiModelProperty(position = 3, value = "The JWT issuer.", example = "thingsboard.io") private String tokenIssuer; /** * Key is used to sign {@link JwtToken}. * Base64 encoded */ + @ApiModelProperty(position = 4, value = "The JWT key is used to sing token. Base64 encoded.", example = "cTU4WnNqemI2aU5wbWVjdm1vYXRzanhjNHRUcXliMjE=") private String tokenSigningKey; - /** - * {@link JwtToken} can be refreshed during this timeframe. - */ - private Integer refreshTokenExpTime; - } diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index 21a18534ed..042453097f 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -136,6 +136,8 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; @@ -286,6 +288,23 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { return restTemplate.postForEntity(baseURL + "/api/admin/securitySettings", securitySettings, SecuritySettings.class).getBody(); } + public Optional getJwtSettings() { + try { + ResponseEntity jwtSettings = restTemplate.getForEntity(baseURL + "/api/admin/jwtSettings", JwtSettings.class); + return Optional.ofNullable(jwtSettings.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + + public JwtPair saveJwtSettings(JwtSettings jwtSettings) { + return restTemplate.postForEntity(baseURL + "/api/admin/jwtSettings", jwtSettings, JwtPair.class).getBody(); + } + public Optional getRepositorySettings() { try { ResponseEntity repositorySettings = restTemplate.getForEntity(baseURL + "/api/admin/repositorySettings", RepositorySettings.class); From 1564ba76f1c46a89c35c565c262148f53248ab24 Mon Sep 17 00:00:00 2001 From: Yuriy Lytvynchuk Date: Tue, 15 Nov 2022 09:38:56 +0200 Subject: [PATCH 48/80] add checkEntityType for init node --- .../engine/transform/TbChangeOriginatorNode.java | 1 + .../engine/util/EntitiesByNameAndTypeLoader.java | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java index 8a801cbb7f..52fa68dba8 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java @@ -127,6 +127,7 @@ public class TbChangeOriginatorNode extends TbAbstractTransformNode { log.error("EntityNamePattern not specified for type [{}]", conf.getEntityType()); throw new IllegalArgumentException("Wrong config for [{}] in TbChangeOriginatorNode!" + ENTITY_SOURCE); } + EntitiesByNameAndTypeLoader.checkEntityType(EntityType.valueOf(conf.getEntityType())); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java index 3c3417ad25..21f2cf5822 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java @@ -20,8 +20,17 @@ import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.id.EntityId; +import java.util.List; + public class EntitiesByNameAndTypeLoader { + private static final List AVAILABLE_ENTITY_TYPES = List.of( + EntityType.DEVICE, + EntityType.ASSET, + EntityType.ENTITY_VIEW, + EntityType.EDGE, + EntityType.USER); + public static EntityId findEntityId(TbContext ctx, EntityType entityType, String entityName) { SearchTextBasedWithAdditionalInfo targetEntity; switch (entityType) { @@ -49,4 +58,10 @@ public class EntitiesByNameAndTypeLoader { return targetEntity.getId(); } + public static void checkEntityType(EntityType entityType) { + if (!AVAILABLE_ENTITY_TYPES.contains(entityType)) { + throw new IllegalStateException("Unexpected entity type " + entityType.name()); + } + } + } From 581af4da0f41cb83ab8b6a9e68099ea0f37b26db Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 15 Nov 2022 10:41:39 +0200 Subject: [PATCH 49/80] UI: Minor improvement style --- .../home/pages/admin/security-settings.component.html | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html index 86f6f9b8c7..e246d9ef40 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html @@ -167,8 +167,8 @@ -
-
+
+
admin.jwt.issuer-name @@ -198,7 +198,7 @@
-
+
admin.jwt.expiration-time Date: Tue, 15 Nov 2022 11:29:12 +0200 Subject: [PATCH 50/80] UI: Minor fix style and fix signing key validation --- .../home/pages/admin/security-settings.component.html | 1 + .../modules/home/pages/admin/security-settings.component.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html index e246d9ef40..39db84d8c6 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html @@ -180,6 +180,7 @@ admin.jwt.signings-key