diff --git a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java index e90ed98351..b4a0e019b6 100644 --- a/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java +++ b/application/src/main/java/org/thingsboard/server/ThingsboardInstallApplication.java @@ -29,6 +29,7 @@ import java.util.Arrays; @ComponentScan({"org.thingsboard.server.install", "org.thingsboard.server.service.component", "org.thingsboard.server.service.install", + "org.thingsboard.server.service.security.auth.jwt.settings", "org.thingsboard.server.dao", "org.thingsboard.server.common.stats", "org.thingsboard.server.common.transport.config.ssl", diff --git a/application/src/main/java/org/thingsboard/server/controller/AdminController.java b/application/src/main/java/org/thingsboard/server/controller/AdminController.java index 772c034f62..29f32b8ac9 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AdminController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AdminController.java @@ -22,7 +22,9 @@ import com.google.common.util.concurrent.MoreExecutors; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.*; import org.springframework.web.context.request.async.DeferredResult; @@ -37,8 +39,13 @@ import org.thingsboard.server.common.data.sms.config.TestSmsRequest; import org.thingsboard.server.common.data.sync.vc.AutoCommitSettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettings; import org.thingsboard.server.common.data.sync.vc.RepositorySettingsInfo; +import org.thingsboard.server.common.data.security.model.JwtSettings; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.dao.settings.AdminSettingsService; import org.thingsboard.server.queue.util.TbCoreComponent; +import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.permission.Operation; import org.thingsboard.server.service.security.permission.Resource; import org.thingsboard.server.service.security.system.SystemSecurityService; @@ -65,6 +72,14 @@ public class AdminController extends BaseController { @Autowired private SystemSecurityService systemSecurityService; + @Lazy + @Autowired + private JwtSettingsService jwtSettingsService; + + @Lazy + @Autowired + private JwtTokenFactory tokenFactory; + @Autowired private EntitiesVersionControlService versionControlService; @@ -152,6 +167,40 @@ public class AdminController extends BaseController { } } + @ApiOperation(value = "Get the JWT Settings object (getJwtSettings)", + notes = "Get the JWT Settings object that contains JWT token policy, etc. " + SYSTEM_AUTHORITY_PARAGRAPH, + produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @RequestMapping(value = "/jwtSettings", method = RequestMethod.GET) + @ResponseBody + public JwtSettings getJwtSettings() throws ThingsboardException { + try { + accessControlService.checkPermission(getCurrentUser(), Resource.ADMIN_SETTINGS, Operation.READ); + return checkNotNull(jwtSettingsService.getJwtSettings()); + } catch (Exception e) { + throw handleException(e); + } + } + + @ApiOperation(value = "Update JWT Settings (saveJwtSettings)", + notes = "Updates the JWT Settings object that contains JWT token policy, etc. The tokenSigningKey field is a Base64 encoded string." + SYSTEM_AUTHORITY_PARAGRAPH, + produces = MediaType.APPLICATION_JSON_VALUE) + @PreAuthorize("hasAuthority('SYS_ADMIN')") + @RequestMapping(value = "/jwtSettings", method = RequestMethod.POST) + @ResponseBody + public JwtPair saveJwtSettings( + @ApiParam(value = "A JSON value representing the JWT Settings.") + @RequestBody JwtSettings jwtSettings) throws ThingsboardException { + try { + SecurityUser securityUser = getCurrentUser(); + accessControlService.checkPermission(securityUser, Resource.ADMIN_SETTINGS, Operation.WRITE); + checkNotNull(jwtSettingsService.saveJwtSettings(jwtSettings)); + return tokenFactory.createTokenPair(securityUser); + } catch (Exception e) { + throw handleException(e); + } + } + @ApiOperation(value = "Send test email (sendTestMail)", notes = "Attempts to send test email to the System Administrator User using Mail Settings provided as a parameter. " + "You may change the 'To' email in the user profile of the System Administrator. " + SYSTEM_AUTHORITY_PARAGRAPH) diff --git a/application/src/main/java/org/thingsboard/server/controller/AuthController.java b/application/src/main/java/org/thingsboard/server/controller/AuthController.java index 8384599c4d..0cb3a3fc92 100644 --- a/application/src/main/java/org/thingsboard/server/controller/AuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/AuthController.java @@ -51,7 +51,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.service.security.model.ActivateUserRequest; import org.thingsboard.server.service.security.model.ChangePasswordRequest; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.ResetPasswordEmailRequest; import org.thingsboard.server.service.security.model.ResetPasswordRequest; import org.thingsboard.server.service.security.model.SecurityUser; @@ -236,7 +236,7 @@ public class AuthController extends BaseController { @RequestMapping(value = "/noauth/activate", method = RequestMethod.POST) @ResponseStatus(value = HttpStatus.OK) @ResponseBody - public JwtTokenPair activateUser( + public JwtPair activateUser( @ApiParam(value = "Activate user request.") @RequestBody ActivateUserRequest activateRequest, @RequestParam(required = false, defaultValue = "true") boolean sendActivationMail, @@ -278,7 +278,7 @@ public class AuthController extends BaseController { @RequestMapping(value = "/noauth/resetPassword", method = RequestMethod.POST) @ResponseStatus(value = HttpStatus.OK) @ResponseBody - public JwtTokenPair resetPassword( + public JwtPair resetPassword( @ApiParam(value = "Reset password request.") @RequestBody ResetPasswordRequest resetPasswordRequest, HttpServletRequest request) throws ThingsboardException { diff --git a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java index 003b4ab450..5ce46e324e 100644 --- a/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java +++ b/application/src/main/java/org/thingsboard/server/controller/TwoFactorAuthController.java @@ -39,7 +39,7 @@ import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; import org.thingsboard.server.service.security.system.SystemSecurityService; @@ -87,8 +87,8 @@ public class TwoFactorAuthController extends BaseController { "and Too Many Requests error if rate limits are exceeded.") @PostMapping("/verification/check") @PreAuthorize("hasAuthority('PRE_VERIFICATION_TOKEN')") - public JwtTokenPair checkTwoFaVerificationCode(@RequestParam TwoFaProviderType providerType, - @RequestParam String verificationCode, HttpServletRequest servletRequest) throws Exception { + public JwtPair checkTwoFaVerificationCode(@RequestParam TwoFaProviderType providerType, + @RequestParam String verificationCode, HttpServletRequest servletRequest) throws Exception { SecurityUser user = getCurrentUser(); boolean verificationSuccess = twoFactorAuthService.checkVerificationCode(user, providerType, verificationCode, true); if (verificationSuccess) { diff --git a/application/src/main/java/org/thingsboard/server/controller/UserController.java b/application/src/main/java/org/thingsboard/server/controller/UserController.java index cea9d2e95a..86e4456638 100644 --- a/application/src/main/java/org/thingsboard/server/controller/UserController.java +++ b/application/src/main/java/org/thingsboard/server/controller/UserController.java @@ -46,7 +46,7 @@ import org.thingsboard.server.common.data.security.UserCredentials; import org.thingsboard.server.common.data.security.event.UserCredentialsInvalidationEvent; import org.thingsboard.server.queue.util.TbCoreComponent; import org.thingsboard.server.service.entitiy.user.TbUserService; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -145,7 +145,7 @@ public class UserController extends BaseController { @PreAuthorize("hasAnyAuthority('SYS_ADMIN', 'TENANT_ADMIN')") @RequestMapping(value = "/user/{userId}/token", method = RequestMethod.GET) @ResponseBody - public JwtTokenPair getUserToken( + public JwtPair getUserToken( @ApiParam(value = USER_ID_PARAM_DESCRIPTION) @PathVariable(USER_ID) String strUserId) throws ThingsboardException { checkParameter(USER_ID, strUserId); @@ -182,7 +182,7 @@ public class UserController extends BaseController { @RequestBody User user, @ApiParam(value = "Send activation email (or use activation link)", defaultValue = "true") @RequestParam(required = false, defaultValue = "true") boolean sendActivationMail, HttpServletRequest request) throws ThingsboardException { - if (Authority.TENANT_ADMIN.equals(getCurrentUser().getAuthority())) { + if (!Authority.SYS_ADMIN.equals(getCurrentUser().getAuthority())) { user.setTenantId(getCurrentUser().getTenantId()); } checkEntity(user.getId(), user, Resource.USER); diff --git a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java index d49cf2dccb..9facea6dde 100644 --- a/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java +++ b/application/src/main/java/org/thingsboard/server/install/ThingsboardInstallService.java @@ -273,6 +273,7 @@ public class ThingsboardInstallService { systemDataLoaderService.createSysAdmin(); systemDataLoaderService.createDefaultTenantProfiles(); systemDataLoaderService.createAdminSettings(); + systemDataLoaderService.createRandomJwtSettings(); systemDataLoaderService.loadSystemWidgets(); systemDataLoaderService.createOAuth2Templates(); systemDataLoaderService.createQueues(); diff --git a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java index 78e03ffe1a..9507912c39 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/DefaultSystemDataLoaderService.java @@ -82,6 +82,7 @@ import org.thingsboard.server.common.data.tenant.profile.DefaultTenantProfileCon import org.thingsboard.server.common.data.tenant.profile.TenantProfileData; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; import org.thingsboard.server.common.data.widget.WidgetsBundle; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.dao.attributes.AttributesService; import org.thingsboard.server.dao.customer.CustomerService; import org.thingsboard.server.dao.device.DeviceCredentialsService; @@ -167,6 +168,9 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { @Autowired private QueueService queueService; + @Autowired + private JwtSettingsService jwtSettingsService; + @Bean protected BCryptPasswordEncoder passwordEncoder() { return new BCryptPasswordEncoder(); @@ -263,6 +267,16 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, mailSettings); } + @Override + public void createRandomJwtSettings() throws Exception { + jwtSettingsService.createRandomJwtSettings(); + } + + @Override + public void saveLegacyYmlSettings() throws Exception { + jwtSettingsService.saveLegacyYmlSettings(); + } + @Override public void createOAuth2Templates() throws Exception { installScripts.createOAuth2Templates(); @@ -656,4 +670,5 @@ public class DefaultSystemDataLoaderService implements SystemDataLoaderService { queueService.saveQueue(sequentialByOriginatorQueue); } } + } diff --git a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java index 1ceb1be289..041351f5e4 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/SystemDataLoaderService.java @@ -23,6 +23,10 @@ public interface SystemDataLoaderService { void createAdminSettings() throws Exception; + void createRandomJwtSettings() throws Exception; + + void saveLegacyYmlSettings() throws Exception; + void createOAuth2Templates() throws Exception; void loadSystemWidgets() throws Exception; diff --git a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java index 93f83414ae..ee34bd7f23 100644 --- a/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java +++ b/application/src/main/java/org/thingsboard/server/service/install/update/DefaultDataUpdateService.java @@ -186,6 +186,7 @@ public class DefaultDataUpdateService implements DataUpdateService { break; case "3.4.1": log.info("Updating data from version 3.4.1 to 3.4.2 ..."); + systemDataLoaderService.saveLegacyYmlSettings(); boolean skipAuditLogsMigration = getEnv("TB_SKIP_AUDIT_LOGS_MIGRATION", false); if (!skipAuditLogsMigration) { log.info("Starting audit logs migration. Can be skipped with TB_SKIP_AUDIT_LOGS_MIGRATION env variable set to true"); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java index ec58ac1aa9..d0af49b1e0 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbCoreConsumerService.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; import org.thingsboard.server.common.msg.rpc.FromDeviceRpcResponse; import org.thingsboard.server.common.stats.StatsFactory; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.queue.util.DataDecodingEncodingService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.gen.transport.TransportProtos; @@ -143,8 +144,9 @@ public class DefaultTbCoreConsumerService extends AbstractConsumerService jwtSettingsService) { + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, tbCoreQueueFactory.createToCoreNotificationsMsgConsumer(), jwtSettingsService); this.mainConsumer = tbCoreQueueFactory.createToCoreMsgConsumer(); this.usageStatsConsumer = tbCoreQueueFactory.createToUsageStatsServiceMsgConsumer(); this.firmwareStatesConsumer = tbCoreQueueFactory.createToOtaPackageStateServiceMsgConsumer(); diff --git a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java index d870af318e..dd97db3703 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/DefaultTbRuleEngineConsumerService.java @@ -70,6 +70,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; @@ -126,7 +127,7 @@ public class DefaultTbRuleEngineConsumerService extends AbstractConsumerService< TbTenantProfileCache tenantProfileCache, TbApiUsageStateService apiUsageStateService, PartitionService partitionService, TbServiceInfoProvider serviceInfoProvider, QueueService queueService) { - super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer()); + super(actorContext, encodingService, tenantProfileCache, deviceProfileCache, assetProfileCache, apiUsageStateService, partitionService, tbRuleEngineQueueFactory.createToRuleEngineNotificationsMsgConsumer(), Optional.empty()); this.statisticsService = statisticsService; this.tbRuleEngineQueueFactory = tbRuleEngineQueueFactory; this.submitStrategyFactory = submitStrategyFactory; diff --git a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java index 47b7f3f9f9..c814ab1704 100644 --- a/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java +++ b/application/src/main/java/org/thingsboard/server/service/queue/processing/AbstractConsumerService.java @@ -33,6 +33,7 @@ import org.thingsboard.server.common.msg.TbActorMsg; import org.thingsboard.server.common.msg.plugin.ComponentLifecycleMsg; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TbCallback; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; import org.thingsboard.server.queue.TbQueueConsumer; import org.thingsboard.server.queue.common.TbProtoQueueMsg; @@ -76,11 +77,13 @@ public abstract class AbstractConsumerService> nfConsumer; + protected final Optional jwtSettingsService; + public AbstractConsumerService(ActorSystemContext actorContext, DataDecodingEncodingService encodingService, TbTenantProfileCache tenantProfileCache, TbDeviceProfileCache deviceProfileCache, TbAssetProfileCache assetProfileCache, TbApiUsageStateService apiUsageStateService, - PartitionService partitionService, TbQueueConsumer> nfConsumer) { + PartitionService partitionService, TbQueueConsumer> nfConsumer, Optional jwtSettingsService) { this.actorContext = actorContext; this.encodingService = encodingService; this.tenantProfileCache = tenantProfileCache; @@ -89,6 +92,7 @@ public abstract class AbstractConsumerService tbClusterService; + private final JwtSettingsValidator jwtSettingsValidator; + + @Value("${security.jwt.tokenExpirationTime:9000}") + private Integer tokenExpirationTime; + @Value("${security.jwt.refreshTokenExpTime:604800}") + private Integer refreshTokenExpTime; + @Value("${security.jwt.tokenIssuer:thingsboard.io}") + private String tokenIssuer; + @Value("${security.jwt.tokenSigningKey:thingsboardDefaultSigningKey}") + private String tokenSigningKey; + + private volatile JwtSettings jwtSettings = null; //lazy init + + /** + * Create JWT admin settings is intended to be called from Install scripts only + */ + @Override + public void createRandomJwtSettings() { + if (getJwtSettingsFromDb() == null) { + log.info("Creating JWT admin settings..."); + this.jwtSettings = getJwtSettingsFromYml(); + if (isSigningKeyDefault(jwtSettings)) { + this.jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString( + RandomStringUtils.randomAlphanumeric(64).getBytes(StandardCharsets.UTF_8))); + } + saveJwtSettings(jwtSettings); + } else { + log.info("Skip creating JWT admin settings because they already exist."); + } + } + + /** + * Create JWT admin settings is intended to be called from Upgrade scripts only + */ + @Override + public void saveLegacyYmlSettings() { + log.info("Saving legacy JWT admin settings from YML..."); + if (getJwtSettingsFromDb() == null) { + saveJwtSettings(getJwtSettingsFromYml()); + } + } + + @Override + public JwtSettings saveJwtSettings(JwtSettings jwtSettings) { + jwtSettingsValidator.validate(jwtSettings); + final AdminSettings adminJwtSettings = mapJwtToAdminSettings(jwtSettings); + final AdminSettings existedSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + if (existedSettings != null) { + adminJwtSettings.setId(existedSettings.getId()); + } + + log.info("Saving new JWT admin settings. From this moment, the JWT parameters from YAML and ENV will be ignored"); + adminSettingsService.saveAdminSettings(TenantId.SYS_TENANT_ID, adminJwtSettings); + + tbClusterService.ifPresent(cs -> cs.broadcastEntityStateChangeEvent(TenantId.SYS_TENANT_ID, TenantId.SYS_TENANT_ID, ComponentLifecycleEvent.UPDATED)); + return reloadJwtSettings(); + } + + @Override + public JwtSettings reloadJwtSettings() { + return getJwtSettings(true); + } + + @Override + public JwtSettings getJwtSettings() { + return getJwtSettings(false); + } + + public JwtSettings getJwtSettings(boolean forceReload) { + if (this.jwtSettings == null || forceReload) { + synchronized (this) { + if (this.jwtSettings == null || forceReload) { + JwtSettings result = getJwtSettingsFromDb(); + if (result == null) { + result = getJwtSettingsFromYml(); + log.warn("Loading the JWT settings from YML since there are no settings in DB. Looks like the upgrade script was not applied."); + } + if (isSigningKeyDefault(result)) { + log.warn("WARNING: The platform is configured to use default JWT Signing Key. " + + "This is a security issue that needs to be resolved. Please change the JWT Signing Key using the Web UI. " + + "Navigate to \"System settings -> Security settings\" while logged in as a System Administrator."); + } + this.jwtSettings = result; + } + } + } + return this.jwtSettings; + } + + private JwtSettings getJwtSettingsFromYml() { + return new JwtSettings(this.tokenExpirationTime, this.refreshTokenExpTime, this.tokenIssuer, this.tokenSigningKey); + } + + private JwtSettings getJwtSettingsFromDb() { + AdminSettings adminJwtSettings = adminSettingsService.findAdminSettingsByKey(TenantId.SYS_TENANT_ID, ADMIN_SETTINGS_JWT_KEY); + return adminJwtSettings != null ? mapAdminToJwtSettings(adminJwtSettings) : null; + } + + private JwtSettings mapAdminToJwtSettings(AdminSettings adminSettings) { + Objects.requireNonNull(adminSettings, "adminSettings for JWT is null"); + return JacksonUtil.treeToValue(adminSettings.getJsonValue(), JwtSettings.class); + } + + private AdminSettings mapJwtToAdminSettings(JwtSettings jwtSettings) { + Objects.requireNonNull(jwtSettings, "jwtSettings is null"); + AdminSettings adminJwtSettings = new AdminSettings(); + adminJwtSettings.setTenantId(TenantId.SYS_TENANT_ID); + adminJwtSettings.setKey(ADMIN_SETTINGS_JWT_KEY); + adminJwtSettings.setJsonValue(JacksonUtil.valueToTree(jwtSettings)); + return adminJwtSettings; + } + + private boolean isSigningKeyDefault(JwtSettings settings) { + return TOKEN_SIGNING_KEY_DEFAULT.equals(settings.getTokenSigningKey()); + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java new file mode 100644 index 0000000000..bf83179b78 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/DefaultJwtSettingsValidator.java @@ -0,0 +1,69 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.jwt.settings; + +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.RandomUtils; +import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.Arrays; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.security.model.JwtSettings; +import org.thingsboard.server.dao.exception.DataValidationException; + +import java.util.Base64; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +@Component +@RequiredArgsConstructor +public class DefaultJwtSettingsValidator implements JwtSettingsValidator { + + @Override + public void validate(JwtSettings jwtSettings) { + if (StringUtils.isEmpty(jwtSettings.getTokenIssuer())) { + throw new DataValidationException("JWT token issuer should be specified!"); + } + if (Optional.ofNullable(jwtSettings.getRefreshTokenExpTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(15)) { + throw new DataValidationException("JWT refresh token expiration time should be at least 15 minutes!"); + } + if (Optional.ofNullable(jwtSettings.getTokenExpirationTime()).orElse(0) <= TimeUnit.MINUTES.toSeconds(1)) { + throw new DataValidationException("JWT token expiration time should be at least 1 minute!"); + } + if (jwtSettings.getTokenExpirationTime() >= jwtSettings.getRefreshTokenExpTime()) { + throw new DataValidationException("JWT token expiration time should greater than JWT refresh token expiration time!"); + } + if (StringUtils.isEmpty(jwtSettings.getTokenSigningKey())) { + throw new DataValidationException("JWT token signing key should be specified!"); + } + + byte[] decodedKey; + try { + decodedKey = Base64.getDecoder().decode(jwtSettings.getTokenSigningKey()); + } catch (Exception e) { + throw new DataValidationException("JWT token signing key should be a valid Base64 encoded string! " + e.getMessage()); + } + + if (Arrays.isNullOrEmpty(decodedKey)) { + throw new DataValidationException("JWT token signing key should be non-empty after Base64 decoding!"); + } + if (decodedKey.length * Byte.SIZE < 256 && !JwtSettingsService.TOKEN_SIGNING_KEY_DEFAULT.equals(jwtSettings.getTokenSigningKey())) { + throw new DataValidationException("JWT token signing key should be a Base64 encoded string representing at least 256 bits of data!"); + } + + System.arraycopy(decodedKey, 0, RandomUtils.nextBytes(decodedKey.length), 0, decodedKey.length); //secure memory + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/InstallJwtSettingsValidator.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/InstallJwtSettingsValidator.java new file mode 100644 index 0000000000..6202e60c96 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/InstallJwtSettingsValidator.java @@ -0,0 +1,39 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.jwt.settings; + +import lombok.RequiredArgsConstructor; +import org.springframework.context.annotation.Primary; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Component; +import org.thingsboard.server.common.data.security.model.JwtSettings; + +/** + * During Install or upgrade the validation is suppressed to keep existing data + * */ + +@Primary +@Profile("install") +@Component +@RequiredArgsConstructor +public class InstallJwtSettingsValidator implements JwtSettingsValidator { + + @Override + public void validate(JwtSettings jwtSettings) { + + } + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java new file mode 100644 index 0000000000..bf02be573c --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsService.java @@ -0,0 +1,35 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.jwt.settings; + +import org.thingsboard.server.common.data.security.model.JwtSettings; + +public interface JwtSettingsService { + + String ADMIN_SETTINGS_JWT_KEY = "jwt"; + String TOKEN_SIGNING_KEY_DEFAULT = "thingsboardDefaultSigningKey"; + + JwtSettings getJwtSettings(); + + JwtSettings reloadJwtSettings(); + + void createRandomJwtSettings(); + + void saveLegacyYmlSettings(); + + JwtSettings saveJwtSettings(JwtSettings jwtSettings); + +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidator.java b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidator.java new file mode 100644 index 0000000000..06a82fc808 --- /dev/null +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/jwt/settings/JwtSettingsValidator.java @@ -0,0 +1,23 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.jwt.settings; + +import org.thingsboard.server.common.data.security.model.JwtSettings; + +public interface JwtSettingsValidator { + + void validate(JwtSettings jwtSettings); +} diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java index 9fd2a680b4..7de6c3fe65 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandler.java @@ -32,7 +32,7 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.oauth2.OAuth2Registration; import org.thingsboard.server.dao.oauth2.OAuth2Service; import org.thingsboard.server.queue.util.TbCoreComponent; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.auth.rest.RestAuthenticationDetails; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -104,10 +104,10 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS SecurityUser securityUser = mapper.getOrCreateUserByClientPrincipal(request, token, oAuth2AuthorizedClient.getAccessToken().getTokenValue(), registration); - JwtTokenPair tokenPair = tokenFactory.createTokenPair(securityUser); - clearAuthenticationAttributes(request, response); - getRedirectStrategy().sendRedirect(request, response, baseUrl + "/?accessToken=" + tokenPair.getToken() + "&refreshToken=" + tokenPair.getRefreshToken()); + + JwtPair tokenPair = tokenFactory.createTokenPair(securityUser); + getRedirectStrategy().sendRedirect(request, response, getRedirectUrl(baseUrl, tokenPair)); systemSecurityService.logLoginAction(securityUser, new RestAuthenticationDetails(request), ActionType.LOGIN, registration.getName(), null); } catch (Exception e) { log.debug("Error occurred during processing authentication success result. " + @@ -128,4 +128,13 @@ public class Oauth2AuthenticationSuccessHandler extends SimpleUrlAuthenticationS super.clearAuthenticationAttributes(request); httpCookieOAuth2AuthorizationRequestRepository.removeAuthorizationRequestCookies(request, response); } + + String getRedirectUrl(String baseUrl, JwtPair tokenPair) { + if (baseUrl.indexOf("?") > 0) { + baseUrl += "&"; + } else { + baseUrl += "/?"; + } + return baseUrl + "accessToken=" + tokenPair.getToken() + "&refreshToken=" + tokenPair.getRefreshToken(); + } } diff --git a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java index 4d7ef01914..f6d9fd8666 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java +++ b/application/src/main/java/org/thingsboard/server/service/security/auth/rest/RestAwareAuthenticationSuccessHandler.java @@ -26,7 +26,7 @@ import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.service.security.auth.MfaAuthenticationToken; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.token.JwtTokenFactory; @@ -49,7 +49,7 @@ public class RestAwareAuthenticationSuccessHandler implements AuthenticationSucc public void onAuthenticationSuccess(HttpServletRequest request, HttpServletResponse response, Authentication authentication) throws IOException, ServletException { SecurityUser securityUser = (SecurityUser) authentication.getPrincipal(); - JwtTokenPair tokenPair = new JwtTokenPair(); + JwtPair tokenPair = new JwtPair(); if (authentication instanceof MfaAuthenticationToken) { int preVerificationTokenLifetime = twoFaConfigManager.getPlatformTwoFaSettings(securityUser.getTenantId(), true) diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java index dca366df18..71fd66231a 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java +++ b/application/src/main/java/org/thingsboard/server/service/security/model/token/JwtTokenFactory.java @@ -24,8 +24,8 @@ import io.jsonwebtoken.MalformedJwtException; import io.jsonwebtoken.SignatureAlgorithm; import io.jsonwebtoken.SignatureException; import io.jsonwebtoken.UnsupportedJwtException; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.security.authentication.BadCredentialsException; import org.springframework.security.core.GrantedAuthority; import org.springframework.stereotype.Component; @@ -35,9 +35,9 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.exception.JwtExpiredTokenException; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; @@ -49,6 +49,7 @@ import java.util.UUID; import java.util.stream.Collectors; @Component +@RequiredArgsConstructor @Slf4j public class JwtTokenFactory { @@ -62,12 +63,7 @@ public class JwtTokenFactory { private static final String CUSTOMER_ID = "customerId"; private static final String SESSION_ID = "sessionId"; - private final JwtSettings settings; - - @Autowired - public JwtTokenFactory(JwtSettings settings) { - this.settings = settings; - } + private final JwtSettingsService jwtSettingsService; /** * Factory method for issuing new JWT Tokens. @@ -80,7 +76,7 @@ public class JwtTokenFactory { UserPrincipal principal = securityUser.getUserPrincipal(); JwtBuilder jwtBuilder = setUpToken(securityUser, securityUser.getAuthorities().stream() - .map(GrantedAuthority::getAuthority).collect(Collectors.toList()), settings.getTokenExpirationTime()); + .map(GrantedAuthority::getAuthority).collect(Collectors.toList()), jwtSettingsService.getJwtSettings().getTokenExpirationTime()); jwtBuilder.claim(FIRST_NAME, securityUser.getFirstName()) .claim(LAST_NAME, securityUser.getLastName()) .claim(ENABLED, securityUser.isEnabled()) @@ -142,7 +138,7 @@ public class JwtTokenFactory { public JwtToken createRefreshToken(SecurityUser securityUser) { UserPrincipal principal = securityUser.getUserPrincipal(); - String token = setUpToken(securityUser, Collections.singletonList(Authority.REFRESH_TOKEN.name()), settings.getRefreshTokenExpTime()) + String token = setUpToken(securityUser, Collections.singletonList(Authority.REFRESH_TOKEN.name()), jwtSettingsService.getJwtSettings().getRefreshTokenExpTime()) .claim(IS_PUBLIC, principal.getType() == UserPrincipal.Type.PUBLIC_ID) .setId(UUID.randomUUID().toString()).compact(); @@ -198,30 +194,30 @@ public class JwtTokenFactory { return Jwts.builder() .setClaims(claims) - .setIssuer(settings.getTokenIssuer()) + .setIssuer(jwtSettingsService.getJwtSettings().getTokenIssuer()) .setIssuedAt(Date.from(currentTime.toInstant())) .setExpiration(Date.from(currentTime.plusSeconds(expirationTime).toInstant())) - .signWith(SignatureAlgorithm.HS512, settings.getTokenSigningKey()); + .signWith(SignatureAlgorithm.HS512, jwtSettingsService.getJwtSettings().getTokenSigningKey()); } public Jws parseTokenClaims(JwtToken token) { try { return Jwts.parser() - .setSigningKey(settings.getTokenSigningKey()) + .setSigningKey(jwtSettingsService.getJwtSettings().getTokenSigningKey()) .parseClaimsJws(token.getToken()); - } catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException | SignatureException ex) { + } catch (UnsupportedJwtException | MalformedJwtException | IllegalArgumentException ex) { log.debug("Invalid JWT Token", ex); throw new BadCredentialsException("Invalid JWT token: ", ex); - } catch (ExpiredJwtException expiredEx) { + } catch (SignatureException | ExpiredJwtException expiredEx) { log.debug("JWT Token is expired", expiredEx); throw new JwtExpiredTokenException(token, "JWT Token expired", expiredEx); } } - public JwtTokenPair createTokenPair(SecurityUser securityUser) { + public JwtPair createTokenPair(SecurityUser securityUser) { JwtToken accessToken = createAccessJwtToken(securityUser); JwtToken refreshToken = createRefreshToken(securityUser); - return new JwtTokenPair(accessToken.getToken(), refreshToken.getToken()); + return new JwtPair(accessToken.getToken(), refreshToken.getToken()); } } diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index 35c613cae6..7fe35d3968 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -107,11 +107,11 @@ plugins: # Security parameters security: # JWT Token parameters - jwt: + jwt: # Since 3.4.2 values are persisted to the database during install or upgrade. On Install, the key will be generated randomly if no custom value set. You can change it later from Web UI under SYS_ADMIN tokenExpirationTime: "${JWT_TOKEN_EXPIRATION_TIME:9000}" # Number of seconds (2.5 hours) refreshTokenExpTime: "${JWT_REFRESH_TOKEN_EXPIRATION_TIME:604800}" # Number of seconds (1 week). tokenIssuer: "${JWT_TOKEN_ISSUER:thingsboard.io}" - tokenSigningKey: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}" + tokenSigningKey: "${JWT_TOKEN_SIGNING_KEY:thingsboardDefaultSigningKey}" # Base64 encoded # Enable/disable access to Tenant Administrators JWT token by System Administrator or Customer Users JWT token by Tenant Administrator user_token_access_enabled: "${SECURITY_USER_TOKEN_ACCESS_ENABLED:true}" # Enable/disable case-sensitive username login @@ -227,6 +227,7 @@ cassandra: default_fetch_size: "${CASSANDRA_DEFAULT_FETCH_SIZE:2000}" # Specify partitioning size for timestamp key-value storage. Example: MINUTES, HOURS, DAYS, MONTHS, INDEFINITE ts_key_value_partitioning: "${TS_KV_PARTITIONING:MONTHS}" + use_ts_key_value_partitioning_on_read: "${USE_TS_KV_PARTITIONING_ON_READ:true}" ts_key_value_partitions_max_cache_size: "${TS_KV_PARTITIONS_MAX_CACHE_SIZE:100000}" ts_key_value_ttl: "${TS_KV_TTL:0}" buffer_size: "${CASSANDRA_QUERY_BUFFER_SIZE:200000}" @@ -625,6 +626,7 @@ mvel: max_black_list_duration_sec: "${MVEL_MAX_BLACKLIST_DURATION_SEC:60}" # Specify thread pool size for javascript executor service thread_pool_size: "${MVEL_THREAD_POOL_SIZE:50}" + compiled_scripts_cache_size: "${MVEL_COMPILED_SCRIPTS_CACHE_SIZE:1000}" stats: enabled: "${TB_MVEL_STATS_ENABLED:false}" print_interval_ms: "${TB_MVEL_STATS_PRINT_INTERVAL_MS:10000}" diff --git a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java index 81ecce8300..285d647a14 100644 --- a/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/BaseAdminControllerTest.java @@ -17,14 +17,21 @@ package org.thingsboard.server.controller; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; import org.junit.Test; import org.mockito.Mockito; import org.springframework.beans.factory.annotation.Autowired; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.MailService; import org.thingsboard.server.common.data.AdminSettings; +import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.service.mail.DefaultMailService; +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import static org.assertj.core.api.Assertions.assertThat; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; @@ -32,8 +39,9 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers. import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; - +@Slf4j public abstract class BaseAdminControllerTest extends AbstractControllerTest { + final JwtSettings defaultJwtSettings = new JwtSettings(9000, 604800, "thingsboard.io", "thingsboardDefaultSigningKey"); @Autowired MailService mailService; @@ -45,67 +53,67 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { public void testFindAdminSettingsByKey() throws Exception { loginSysAdmin(); doGet("/api/admin/settings/general") - .andExpect(status().isOk()) - .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$.id", notNullValue())) - .andExpect(jsonPath("$.key", is("general"))) - .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://localhost:8080"))); - + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.id", notNullValue())) + .andExpect(jsonPath("$.key", is("general"))) + .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://localhost:8080"))); + doGet("/api/admin/settings/mail") - .andExpect(status().isOk()) - .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$.id", notNullValue())) - .andExpect(jsonPath("$.key", is("mail"))) - .andExpect(jsonPath("$.jsonValue.smtpProtocol", is("smtp"))) - .andExpect(jsonPath("$.jsonValue.smtpHost", is("localhost"))) - .andExpect(jsonPath("$.jsonValue.smtpPort", is("25"))); - + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.id", notNullValue())) + .andExpect(jsonPath("$.key", is("mail"))) + .andExpect(jsonPath("$.jsonValue.smtpProtocol", is("smtp"))) + .andExpect(jsonPath("$.jsonValue.smtpHost", is("localhost"))) + .andExpect(jsonPath("$.jsonValue.smtpPort", is("25"))); + doGet("/api/admin/settings/unknown") - .andExpect(status().isNotFound()); - + .andExpect(status().isNotFound()); + } - + @Test public void testSaveAdminSettings() throws Exception { loginSysAdmin(); - AdminSettings adminSettings = doGet("/api/admin/settings/general", AdminSettings.class); - + AdminSettings adminSettings = doGet("/api/admin/settings/general", AdminSettings.class); + JsonNode jsonValue = adminSettings.getJsonValue(); ((ObjectNode) jsonValue).put("baseUrl", "http://myhost.org"); adminSettings.setJsonValue(jsonValue); doPost("/api/admin/settings", adminSettings).andExpect(status().isOk()); - + doGet("/api/admin/settings/general") - .andExpect(status().isOk()) - .andExpect(content().contentType(contentType)) - .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://myhost.org"))); - + .andExpect(status().isOk()) + .andExpect(content().contentType(contentType)) + .andExpect(jsonPath("$.jsonValue.baseUrl", is("http://myhost.org"))); + ((ObjectNode) jsonValue).put("baseUrl", "http://localhost:8080"); adminSettings.setJsonValue(jsonValue); - + doPost("/api/admin/settings", adminSettings) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } @Test public void testSaveAdminSettingsWithEmptyKey() throws Exception { loginSysAdmin(); - AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); + AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); adminSettings.setKey(null); doPost("/api/admin/settings", adminSettings) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("Key should be specified"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("Key should be specified"))); } - + @Test public void testChangeAdminSettingsKey() throws Exception { loginSysAdmin(); - AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); + AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); adminSettings.setKey("newKey"); doPost("/api/admin/settings", adminSettings) - .andExpect(status().isBadRequest()) - .andExpect(statusReason(containsString("is prohibited"))); + .andExpect(status().isBadRequest()) + .andExpect(statusReason(containsString("is prohibited"))); } @Test @@ -113,7 +121,7 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { loginSysAdmin(); AdminSettings adminSettings = doGet("/api/admin/settings/mail", AdminSettings.class); doPost("/api/admin/settings/testMail", adminSettings) - .andExpect(status().isOk()); + .andExpect(status().isOk()); } @Test @@ -139,4 +147,48 @@ public abstract class BaseAdminControllerTest extends AbstractControllerTest { doPost("/api/admin/settings/testMail", adminSettings).andExpect(status().is5xxServerError()); Mockito.doNothing().when(mailService).sendTestMail(Mockito.any(), Mockito.any()); } + + void resetJwtSettingsToDefault() throws Exception { + loginSysAdmin(); + doPost("/api/admin/jwtSettings", defaultJwtSettings).andExpect(status().isOk()); // jwt test scenarios are always started from + loginTenantAdmin(); + } + + @Test + public void testGetAndSaveDefaultJwtSettings() throws Exception { + JwtSettings jwtSettings; + loginSysAdmin(); + + jwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(defaultJwtSettings); + + doPost("/api/admin/jwtSettings", jwtSettings).andExpect(status().isOk()); + + jwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(defaultJwtSettings); + + resetJwtSettingsToDefault(); + } + + @Test + public void testCreateJwtSettings() throws Exception { + loginSysAdmin(); + + JwtSettings jwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(defaultJwtSettings); + + jwtSettings.setTokenSigningKey(Base64.getEncoder().encodeToString( + RandomStringUtils.randomAlphanumeric(256 / Byte.SIZE).getBytes(StandardCharsets.UTF_8))); + + doPost("/api/admin/jwtSettings", jwtSettings).andExpect(status().isOk()); + + doGet("/api/admin/jwtSettings").andExpect(status().isUnauthorized()); //the old JWT token does not work after signing key was changed! + + loginSysAdmin(); + JwtSettings newJwtSettings = doGet("/api/admin/jwtSettings", JwtSettings.class); + assertThat(jwtSettings).isEqualTo(newJwtSettings); + + resetJwtSettingsToDefault(); + } + } diff --git a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java index 9839843b0c..6c7dfc3cd7 100644 --- a/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java +++ b/application/src/test/java/org/thingsboard/server/controller/TwoFactorAuthTest.java @@ -51,7 +51,7 @@ import org.thingsboard.server.dao.user.UserService; import org.thingsboard.server.service.security.auth.mfa.TwoFactorAuthService; import org.thingsboard.server.service.security.auth.mfa.config.TwoFaConfigManager; import org.thingsboard.server.service.security.auth.rest.LoginRequest; -import org.thingsboard.server.service.security.model.JwtTokenPair; +import org.thingsboard.server.common.data.security.model.JwtPair; import java.time.Duration; import java.util.Arrays; @@ -396,7 +396,7 @@ public abstract class TwoFactorAuthTest extends AbstractControllerTest { private void logInWithPreVerificationToken(String username, String password) throws Exception { LoginRequest loginRequest = new LoginRequest(username, password); - JwtTokenPair response = readResponse(doPost("/api/auth/login", loginRequest).andExpect(status().isOk()), JwtTokenPair.class); + JwtPair response = readResponse(doPost("/api/auth/login", loginRequest).andExpect(status().isOk()), JwtPair.class); assertThat(response.getToken()).isNotNull(); assertThat(response.getRefreshToken()).isNull(); assertThat(response.getScope()).isEqualTo(Authority.PRE_VERIFICATION_TOKEN); diff --git a/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java b/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java index a40a087514..1c3c134a82 100644 --- a/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java +++ b/application/src/test/java/org/thingsboard/server/service/script/MvelInvokeServiceTest.java @@ -16,6 +16,7 @@ package org.thingsboard.server.service.script; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.github.benmanes.caffeine.cache.Cache; import org.junit.Assert; import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; @@ -24,15 +25,22 @@ import org.springframework.test.context.TestPropertySource; import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.script.api.ScriptType; import org.thingsboard.script.api.mvel.MvelInvokeService; +import org.thingsboard.script.api.mvel.MvelScript; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.controller.AbstractControllerTest; import org.thingsboard.server.dao.service.DaoSqlTest; +import java.io.Serializable; +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @DaoSqlTest @@ -41,6 +49,7 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy; "mvel.max_total_args_size=50", "mvel.max_result_size=50", "mvel.max_errors=2", + "mvel.compiled_scripts_cache_size=100" }) class MvelInvokeServiceTest extends AbstractControllerTest { @@ -110,6 +119,89 @@ class MvelInvokeServiceTest extends AbstractControllerTest { assertThatScriptIsBlocked(scriptId); } + @Test + void givenScriptsWithSameBody_thenCompileAndCacheOnlyOnce() throws Exception { + String script = "return msg.temperature > 20;"; + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + } + + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Map scriptMap = getFieldValue(invokeService, "scriptMap"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + String scriptHash = scriptIdToHash.get(scriptsIds.get(0)); + + assertThat(scriptsIds.stream().map(scriptIdToHash::get)).containsOnly(scriptHash); + assertThat(scriptMap).containsKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNotNull(); + } + + @Test + public void whenReleasingScript_thenCheckForScriptHashUsages() throws Exception { + String script = "return msg.temperature > 20;"; + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + } + + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Map scriptMap = getFieldValue(invokeService, "scriptMap"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + String scriptHash = scriptIdToHash.get(scriptsIds.get(0)); + for (int i = 0; i < 9; i++) { + UUID scriptId = scriptsIds.get(i); + assertThat(scriptIdToHash).containsKey(scriptId); + invokeService.release(scriptId); + assertThat(scriptIdToHash).doesNotContainKey(scriptId); + } + assertThat(scriptMap).containsKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNotNull(); + + invokeService.release(scriptsIds.get(9)); + assertThat(scriptMap).doesNotContainKey(scriptHash); + assertThat(compiledScriptsCache.getIfPresent(scriptHash)).isNull(); + } + + @Test + public void whenCompiledScriptsCacheIsTooBig_thenRemoveRarelyUsedScripts() throws Exception { + Map scriptIdToHash = getFieldValue(invokeService, "scriptIdToHash"); + Cache compiledScriptsCache = getFieldValue(invokeService, "compiledScriptsCache"); + + List scriptsIds = new ArrayList<>(); + for (int i = 0; i < 110; i++) { // mvel.compiled_scripts_cache_size = 100 + String script = "return msg.temperature > " + i; + UUID scriptId = evalScript(script); + scriptsIds.add(scriptId); + + for (int j = 0; j < i; j++) { + invokeScript(scriptId, "{ \"temperature\": 12 }"); // so that scriptsIds is ordered by number of invocations + } + } + + ConcurrentMap cache = compiledScriptsCache.asMap(); + + for (int i = 0; i < 10; i++) { // iterating rarely used scripts + UUID scriptId = scriptsIds.get(i); + String scriptHash = scriptIdToHash.get(scriptId); + assertThat(cache).doesNotContainKey(scriptHash); + } + for (int i = 10; i < 110; i++) { + UUID scriptId = scriptsIds.get(i); + String scriptHash = scriptIdToHash.get(scriptId); + assertThat(cache).containsKey(scriptHash); + } + + UUID scriptRemovedFromCache = scriptsIds.get(0); + assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNull(); + invokeScript(scriptRemovedFromCache, "{ \"temperature\": 12 }"); + assertThat(compiledScriptsCache.getIfPresent(scriptIdToHash.get(scriptRemovedFromCache))).isNotNull(); + } + private void assertThatScriptIsBlocked(UUID scriptId) { assertThatThrownBy(() -> { invokeScript(scriptId, "{}"); @@ -125,4 +217,10 @@ class MvelInvokeServiceTest extends AbstractControllerTest { return invokeService.invokeScript(TenantId.SYS_TENANT_ID, null, scriptId, msg, "{}", "POST_TELEMETRY_REQUEST").get().toString(); } + private T getFieldValue(Object target, String fieldName) throws Exception { + Field field = target.getClass().getDeclaredField(fieldName); + field.setAccessible(true); + return (T) field.get(target); + } + } diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java index f865c9b5e0..bf89eadda5 100644 --- a/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/JwtTokenFactoryTest.java @@ -23,7 +23,8 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.id.UserId; import org.thingsboard.server.common.data.security.Authority; import org.thingsboard.server.common.data.security.model.JwtToken; -import org.thingsboard.server.config.JwtSettings; +import org.thingsboard.server.common.data.security.model.JwtSettings; +import org.thingsboard.server.service.security.auth.jwt.settings.JwtSettingsService; import org.thingsboard.server.service.security.model.SecurityUser; import org.thingsboard.server.service.security.model.UserPrincipal; import org.thingsboard.server.service.security.model.token.AccessJwtToken; @@ -36,6 +37,8 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.BDDMockito.willReturn; +import static org.mockito.Mockito.mock; public class JwtTokenFactoryTest { @@ -50,7 +53,10 @@ public class JwtTokenFactoryTest { jwtSettings.setTokenExpirationTime((int) TimeUnit.HOURS.toSeconds(2)); jwtSettings.setRefreshTokenExpTime((int) TimeUnit.DAYS.toSeconds(7)); - tokenFactory = new JwtTokenFactory(jwtSettings); + JwtSettingsService jwtSettingsService = mock(JwtSettingsService.class); + willReturn(jwtSettings).given(jwtSettingsService).getJwtSettings(); + + tokenFactory = new JwtTokenFactory(jwtSettingsService); } @Test diff --git a/application/src/test/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandlerTest.java b/application/src/test/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandlerTest.java new file mode 100644 index 0000000000..9b1e356b75 --- /dev/null +++ b/application/src/test/java/org/thingsboard/server/service/security/auth/oauth2/Oauth2AuthenticationSuccessHandlerTest.java @@ -0,0 +1,68 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.service.security.auth.oauth2; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.springframework.beans.factory.annotation.Autowired; +import org.thingsboard.server.common.data.id.UserId; +import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.controller.AbstractControllerTest; +import org.thingsboard.server.dao.service.DaoSqlTest; +import org.thingsboard.server.service.security.model.SecurityUser; +import org.thingsboard.server.service.security.model.token.JwtTokenFactory; + +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; + +@DaoSqlTest +public class Oauth2AuthenticationSuccessHandlerTest extends AbstractControllerTest { + + @Autowired + private Oauth2AuthenticationSuccessHandler oauth2AuthenticationSuccessHandler; + + @Mock + private JwtTokenFactory jwtTokenFactory; + + private SecurityUser securityUser; + + @Before + public void before() { + UserId userId = new UserId(UUID.randomUUID()); + securityUser = new SecurityUser(userId); + when(jwtTokenFactory.createTokenPair(eq(securityUser))).thenReturn(new JwtPair("testAccessToken", "testRefreshToken")); + } + + @Test + public void testGetRedirectUrl() { + JwtPair jwtPair = jwtTokenFactory.createTokenPair(securityUser); + + String urlWithoutParams = "http://localhost:8080/dashboardGroups/3fa13530-6597-11ed-bd76-8bd591f0ec3e"; + String urlWithParams = "http://localhost:8080/dashboardGroups/3fa13530-6597-11ed-bd76-8bd591f0ec3e?state=someState&page=1"; + + String redirectUrl = oauth2AuthenticationSuccessHandler.getRedirectUrl(urlWithoutParams, jwtPair); + String expectedUrl = urlWithoutParams + "/?accessToken=" + jwtPair.getToken() + "&refreshToken=" + jwtPair.getRefreshToken(); + assertEquals(expectedUrl, redirectUrl); + + redirectUrl = oauth2AuthenticationSuccessHandler.getRedirectUrl(urlWithParams, jwtPair); + expectedUrl = urlWithParams + "&accessToken=" + jwtPair.getToken() + "&refreshToken=" + jwtPair.getRefreshToken(); + assertEquals(expectedUrl, redirectUrl); + } +} \ No newline at end of file diff --git a/application/src/test/resources/logback-test.xml b/application/src/test/resources/logback-test.xml index d3301bf660..3762c8aa7c 100644 --- a/application/src/test/resources/logback-test.xml +++ b/application/src/test/resources/logback-test.xml @@ -16,6 +16,8 @@ + + diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SearchTextBasedWithAdditionalInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/SearchTextBasedWithAdditionalInfo.java index 3545b290bd..8d7fba5238 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SearchTextBasedWithAdditionalInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SearchTextBasedWithAdditionalInfo.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.thingsboard.server.common.data.id.UUIDBased; +import org.thingsboard.server.common.data.validation.NoXss; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -36,6 +37,7 @@ import java.util.function.Consumer; public abstract class SearchTextBasedWithAdditionalInfo extends SearchTextBased implements HasAdditionalInfo { public static final ObjectMapper mapper = new ObjectMapper(); + @NoXss private transient JsonNode additionalInfo; @JsonIgnore private byte[] additionalInfoBytes; diff --git a/application/src/main/java/org/thingsboard/server/service/security/model/JwtTokenPair.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtPair.java similarity index 85% rename from application/src/main/java/org/thingsboard/server/service/security/model/JwtTokenPair.java rename to common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtPair.java index 02e28cd885..eb50a11a92 100644 --- a/application/src/main/java/org/thingsboard/server/service/security/model/JwtTokenPair.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtPair.java @@ -13,19 +13,18 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.service.security.model; +package org.thingsboard.server.common.data.security.model; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import org.thingsboard.server.common.data.security.Authority; -@ApiModel(value = "JWT Token Pair") +@ApiModel(value = "JWT Pair") @Data @NoArgsConstructor -public class JwtTokenPair { +public class JwtPair { @ApiModelProperty(position = 1, value = "The JWT Access Token. Used to perform API calls.", example = "AAB254FF67D..") private String token; @@ -34,7 +33,7 @@ public class JwtTokenPair { private Authority scope; - public JwtTokenPair(String token, String refreshToken) { + public JwtPair(String token, String refreshToken) { this.token = token; this.refreshToken = refreshToken; } diff --git a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java similarity index 58% rename from application/src/main/java/org/thingsboard/server/config/JwtSettings.java rename to common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java index 95e510612a..f5668ff088 100644 --- a/application/src/main/java/org/thingsboard/server/config/JwtSettings.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/security/model/JwtSettings.java @@ -13,35 +13,43 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.thingsboard.server.config; +package org.thingsboard.server.common.data.security.model; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; -import org.thingsboard.server.common.data.security.model.JwtToken; +import lombok.NoArgsConstructor; -@Component -@ConfigurationProperties(prefix = "security.jwt") +@ApiModel(value = "JWT Settings") +@AllArgsConstructor +@NoArgsConstructor @Data public class JwtSettings { + /** * {@link JwtToken} will expire after this time. */ + @ApiModelProperty(position = 1, value = "The JWT will expire after seconds.", example = "9000") private Integer tokenExpirationTime; + /** + * {@link JwtToken} can be refreshed during this timeframe. + */ + @ApiModelProperty(position = 2, value = "The JWT can be refreshed during seconds.", example = "604800") + private Integer refreshTokenExpTime; + /** * Token issuer. */ + @ApiModelProperty(position = 3, value = "The JWT issuer.", example = "thingsboard.io") private String tokenIssuer; /** * Key is used to sign {@link JwtToken}. + * Base64 encoded */ + @ApiModelProperty(position = 4, value = "The JWT key is used to sing token. Base64 encoded.", example = "cTU4WnNqemI2aU5wbWVjdm1vYXRzanhjNHRUcXliMjE=") private String tokenSigningKey; - /** - * {@link JwtToken} can be refreshed during this timeframe. - */ - private Integer refreshTokenExpTime; - } diff --git a/common/script/script-api/pom.xml b/common/script/script-api/pom.xml index 62dc00ca97..b7ae4993f9 100644 --- a/common/script/script-api/pom.xml +++ b/common/script/script-api/pom.xml @@ -56,6 +56,10 @@ com.google.code.gson gson + + com.github.ben-manes.caffeine + caffeine + org.slf4j slf4j-api diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java index 901a49180e..4fb029096b 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/DefaultMvelInvokeService.java @@ -15,6 +15,10 @@ */ package org.thingsboard.script.api.mvel; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -42,12 +46,15 @@ import org.thingsboard.server.common.stats.TbApiUsageStateClient; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.Map; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; @Slf4j @@ -55,7 +62,10 @@ import java.util.regex.Pattern; @Service public class DefaultMvelInvokeService extends AbstractScriptInvokeService implements MvelInvokeService { - protected Map scriptMap = new ConcurrentHashMap<>(); + protected final Map scriptIdToHash = new ConcurrentHashMap<>(); + protected final Map scriptMap = new ConcurrentHashMap<>(); + protected Cache compiledScriptsCache; + private SandboxedParserConfiguration parserConfig; private static final Pattern NEW_KEYWORD_PATTERN = Pattern.compile("new\\s"); @@ -92,8 +102,13 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Value("${mvel.max_memory_limit_mb:8}") private long maxMemoryLimitMb; + @Value("${mvel.compiled_scripts_cache_size:1000}") + private int compiledScriptsCacheSize; + private ListeningExecutorService executor; + private final Lock lock = new ReentrantLock(); + protected DefaultMvelInvokeService(Optional apiUsageStateClient, Optional apiUsageReportClient) { super(apiUsageStateClient, apiUsageReportClient); } @@ -115,11 +130,14 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem executor = MoreExecutors.listeningDecorator(ThingsBoardExecutors.newWorkStealingPool(threadPoolSize, "mvel-executor")); try { // Special command to warm up MVEL engine - Serializable script = MVEL.compileExpression("var warmUp = {}; warmUp", new SandboxedParserContext(parserConfig)); + Serializable script = compileScript("var warmUp = {}; warmUp"); MVEL.executeTbExpression(script, new ExecutionContext(parserConfig), Collections.emptyMap()); } catch (Exception e) { // do nothing } + compiledScriptsCache = Caffeine.newBuilder() + .maximumSize(compiledScriptsCacheSize) + .build(); } @PreDestroy @@ -141,16 +159,26 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Override protected boolean isScriptPresent(UUID scriptId) { - return scriptMap.containsKey(scriptId); + return scriptIdToHash.containsKey(scriptId); } @Override protected ListenableFuture doEvalScript(TenantId tenantId, ScriptType scriptType, String scriptBody, UUID scriptId, String[] argNames) { return executor.submit(() -> { try { - Serializable compiledScript = MVEL.compileExpression(scriptBody, new SandboxedParserContext(parserConfig)); - MvelScript script = new MvelScript(compiledScript, scriptBody, argNames); - scriptMap.put(scriptId, script); + String scriptHash = hash(scriptBody, argNames); + compiledScriptsCache.get(scriptHash, k -> { + return compileScript(scriptBody); + }); + lock.lock(); + try { + scriptIdToHash.put(scriptId, scriptHash); + scriptMap.computeIfAbsent(scriptHash, k -> { + return new MvelScript(scriptBody, argNames); + }); + } finally { + lock.unlock(); + } return scriptId; } catch (Exception e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.COMPILATION, scriptBody, e); @@ -162,12 +190,16 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem protected MvelScriptExecutionTask doInvokeFunction(UUID scriptId, Object[] args) { ExecutionContext executionContext = new ExecutionContext(this.parserConfig, maxMemoryLimitMb * 1024 * 1024); return new MvelScriptExecutionTask(executionContext, executor.submit(() -> { - MvelScript script = scriptMap.get(scriptId); - if (script == null) { + String scriptHash = scriptIdToHash.get(scriptId); + if (scriptHash == null) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.OTHER, null, new RuntimeException("Script not found!")); } + MvelScript script = scriptMap.get(scriptHash); + Serializable compiledScript = compiledScriptsCache.get(scriptHash, k -> { + return compileScript(script.getScriptBody()); + }); try { - return MVEL.executeTbExpression(script.getCompiledScript(), executionContext, script.createVars(args)); + return MVEL.executeTbExpression(compiledScript, executionContext, script.createVars(args)); } catch (ScriptMemoryOverflowException e) { throw new TbScriptException(scriptId, TbScriptException.ErrorCode.OTHER, script.getScriptBody(), new RuntimeException("Script memory overflow!")); } catch (Exception e) { @@ -178,6 +210,32 @@ public class DefaultMvelInvokeService extends AbstractScriptInvokeService implem @Override protected void doRelease(UUID scriptId) throws Exception { - scriptMap.remove(scriptId); + String scriptHash = scriptIdToHash.remove(scriptId); + if (scriptHash != null) { + lock.lock(); + try { + if (!scriptIdToHash.containsValue(scriptHash)) { + scriptMap.remove(scriptHash); + compiledScriptsCache.invalidate(scriptHash); + } + } finally { + lock.unlock(); + } + } } + + private Serializable compileScript(String scriptBody) { + return MVEL.compileExpression(scriptBody, new SandboxedParserContext(parserConfig)); + } + + @SuppressWarnings("UnstableApiUsage") + protected String hash(String scriptBody, String[] argNames) { + Hasher hasher = Hashing.murmur3_128().newHasher(); + hasher.putUnencodedChars(scriptBody); + for (String argName : argNames) { + hasher.putString(argName, StandardCharsets.UTF_8); + } + return hasher.hash().toString(); + } + } diff --git a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java index 7a84c7b0af..bca5d8d546 100644 --- a/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java +++ b/common/script/script-api/src/main/java/org/thingsboard/script/api/mvel/MvelScript.java @@ -24,7 +24,6 @@ import java.util.Map; @Data public class MvelScript { - private final Serializable compiledScript; private final String scriptBody; private final String[] argNames; diff --git a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java index 420739e37c..1d11d34b2d 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/rule/BaseRuleChainService.java @@ -731,6 +731,8 @@ public class BaseRuleChainService extends AbstractEntityService implements RuleC ConstraintViolationException e = extractConstraintViolationException(t).orElse(null); if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("fk_default_rule_chain_device_profile")) { throw new DataValidationException("The rule chain referenced by the device profiles cannot be deleted!"); + } else if (e != null && e.getConstraintName() != null && e.getConstraintName().equalsIgnoreCase("fk_default_rule_chain_asset_profile")) { + throw new DataValidationException("The rule chain referenced by the asset profiles cannot be deleted!"); } else { throw t; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java index 5e61d75fea..31496b84d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/NoXssValidator.java @@ -15,6 +15,7 @@ */ package org.thingsboard.server.dao.service; +import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; import org.owasp.validator.html.AntiSamy; import org.owasp.validator.html.Policy; @@ -48,12 +49,18 @@ public class NoXssValidator implements ConstraintValidator { @Override public boolean isValid(Object value, ConstraintValidatorContext constraintValidatorContext) { - if (!(value instanceof String) || ((String) value).isEmpty()) { + String stringValue; + if (value instanceof CharSequence || value instanceof JsonNode) { + stringValue = value.toString(); + } else { + return true; + } + if (stringValue.isEmpty()) { return true; } try { - return xssChecker.scan((String) value, xssPolicy).getNumberOfErrors() == 0; + return xssChecker.scan(stringValue, xssPolicy).getNumberOfErrors() == 0; } catch (ScanException | PolicyException e) { return false; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java index 4df37ddc00..5a26990fcd 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/UserDataValidator.java @@ -19,6 +19,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.Customer; +import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.User; @@ -66,6 +67,24 @@ public class UserDataValidator extends DataValidator { } } + @Override + protected User validateUpdate(TenantId tenantId, User user) { + User old = userDao.findById(user.getTenantId(), user.getId().getId()); + if (old == null) { + throw new DataValidationException("Can't update non existing user!"); + } + if (!old.getTenantId().equals(user.getTenantId())) { + throw new DataValidationException("Can't update user tenant id!"); + } + if (!old.getAuthority().equals(user.getAuthority())) { + throw new DataValidationException("Can't update user authority!"); + } + if (!old.getCustomerId().equals(user.getCustomerId())) { + throw new DataValidationException("Can't update user customer id!"); + } + return old; + } + @Override protected void validateDataImpl(TenantId requestTenantId, User user) { if (StringUtils.isEmpty(user.getEmail())) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java index 69cb4a2d86..d91a8c5fc4 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/query/DefaultAlarmQueryRepository.java @@ -140,6 +140,7 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { selectPart.append(" a.originator_id as entity_id "); } EntityDataSortOrder sortOrder = pageLink.getSortOrder(); + String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch()); if (sortOrder != null && sortOrder.getKey().getType().equals(EntityKeyType.ALARM_FIELD)) { String sortOrderKey = sortOrder.getKey().getKey(); sortPart.append(alarmFieldColumnMap.getOrDefault(sortOrderKey, sortOrderKey)) @@ -166,7 +167,11 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } joinPart.append(" as e(id, priority)) e "); if (pageLink.isSearchPropagatedAlarms()) { - joinPart.append("on ea.entity_id = e.id"); + if (textSearchQuery.isEmpty()) { + joinPart.append("on ea.entity_id = e.id"); + } else { + joinPart.append("on a.entity_id = e.id"); + } } else { joinPart.append("on a.originator_id = e.id"); } @@ -230,13 +235,11 @@ public class DefaultAlarmQueryRepository implements AlarmQueryRepository { } } - String textSearchQuery = buildTextSearchQuery(ctx, query.getAlarmFields(), pageLink.getTextSearch()); - String mainQuery; - if (!textSearchQuery.isEmpty()) { - mainQuery = selectPart.toString() + fromPart.toString() + wherePart.toString(); - mainQuery = String.format("select * from (%s) a %s WHERE %s", mainQuery, joinPart, textSearchQuery); + String mainQuery = String.format("%s%s", selectPart, fromPart); + if (textSearchQuery.isEmpty()) { + mainQuery = String.format("%s%s%s", mainQuery, joinPart, wherePart); } else { - mainQuery = selectPart.toString() + fromPart.toString() + joinPart.toString() + wherePart.toString(); + mainQuery = String.format("select * from (%s%s) a %s WHERE %s", mainQuery, wherePart, joinPart, textSearchQuery); } String countQuery = String.format("select count(*) from (%s) result", mainQuery); long queryTs = System.currentTimeMillis(); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java index cf726c32c4..4875468c09 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/AbstractChunkedAggregationTimeseriesDao.java @@ -105,11 +105,6 @@ public abstract class AbstractChunkedAggregationTimeseriesDao extends AbstractSq return Futures.immediateFuture(null); } - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return Futures.immediateFuture(null); - } - @Override public ListenableFuture> findAllAsync(TenantId tenantId, EntityId entityId, List queries) { return processFindAllAsync(tenantId, entityId, queries); diff --git a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java index c9daf1ec6d..08089a818e 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sqlts/timescale/TimescaleTimeseriesDao.java @@ -18,7 +18,6 @@ package org.thingsboard.server.dao.sqlts.timescale; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.util.concurrent.SettableFuture; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; @@ -36,7 +35,6 @@ import org.thingsboard.server.common.stats.StatsFactory; import org.thingsboard.server.dao.DaoUtil; import org.thingsboard.server.dao.model.sql.AbstractTsKvEntity; import org.thingsboard.server.dao.model.sqlts.timescale.ts.TimescaleTsKvEntity; -import org.thingsboard.server.dao.model.sqlts.ts.TsKvEntity; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueParams; import org.thingsboard.server.dao.sql.TbSqlBlockingQueueWrapper; import org.thingsboard.server.dao.sqlts.AbstractSqlTimeseriesDao; @@ -52,7 +50,6 @@ import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.CompletableFuture; import java.util.function.Function; @Component @@ -144,11 +141,6 @@ public class TimescaleTimeseriesDao extends AbstractSqlTimeseriesDao implements }); } - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - return service.submit(() -> null); - } - @Override public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java index c8a5076c65..a2905a7a64 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/BaseTimeseriesService.java @@ -46,7 +46,6 @@ import org.thingsboard.server.dao.service.Validator; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -277,7 +276,6 @@ public class BaseTimeseriesService implements TimeseriesService { private void deleteAndRegisterFutures(TenantId tenantId, List> futures, EntityId entityId, DeleteTsKvQuery query) { futures.add(Futures.transform(timeseriesDao.remove(tenantId, entityId, query), v -> null, MoreExecutors.directExecutor())); futures.add(timeseriesLatestDao.removeLatest(tenantId, entityId, query)); - futures.add(Futures.transform(timeseriesDao.removePartition(tenantId, entityId, query), v -> null, MoreExecutors.directExecutor())); } private static void validate(EntityId entityId) { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java index 3d704a0429..737ef8d8f6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDao.java @@ -28,6 +28,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -59,7 +60,6 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -82,17 +82,21 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD protected static final int MIN_AGGREGATION_STEP_MS = 1000; public static final String ASC_ORDER = "ASC"; public static final long SECONDS_IN_DAY = TimeUnit.DAYS.toSeconds(1); - - protected static List FIXED_PARTITION = Arrays.asList(new Long[]{0L}); + protected static final List FIXED_PARTITION = List.of(0L); private CassandraTsPartitionsCache cassandraTsPartitionsCache; @Autowired private Environment environment; + @Getter @Value("${cassandra.query.ts_key_value_partitioning}") private String partitioning; + @Getter + @Value("${cassandra.query.use_ts_key_value_partitioning_on_read:true}") + private boolean useTsKeyValuePartitioningOnRead; + @Value("${cassandra.query.ts_key_value_partitions_max_cache_size:100000}") private long partitionsCacheSize; @@ -222,46 +226,6 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD return resultFuture; } - @Override - public ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query) { - long minPartition = toPartitionTs(query.getStartTs()); - long maxPartition = toPartitionTs(query.getEndTs()); - if (minPartition == maxPartition) { - return Futures.immediateFuture(null); - } else { - TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); - - final SimpleListenableFuture resultFuture = new SimpleListenableFuture<>(); - final ListenableFuture> partitionsListFuture = Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); - - Futures.addCallback(partitionsListFuture, new FutureCallback>() { - @Override - public void onSuccess(@Nullable List partitions) { - int index = 0; - if (minPartition != query.getStartTs()) { - index = 1; - } - List partitionsToDelete = new ArrayList<>(); - for (int i = index; i < partitions.size() - 1; i++) { - partitionsToDelete.add(partitions.get(i)); - } - QueryCursor cursor = new QueryCursor(entityId.getEntityType().name(), entityId.getId(), query, partitionsToDelete); - deletePartitionAsync(tenantId, cursor, resultFuture); - - for (Long partition : partitionsToDelete) { - cassandraTsPartitionsCache.invalidate(new CassandraPartitionCacheKey(entityId, query.getKey(), partition)); - } - } - - @Override - public void onFailure(Throwable t) { - log.error("[{}][{}] Failed to fetch partitions for interval {}-{}", entityId.getEntityType().name(), entityId.getId(), minPartition, maxPartition, t); - } - }, readResultsProcessingExecutor); - return resultFuture; - } - } - @Override public ListenableFuture findAllAsync(TenantId tenantId, EntityId entityId, ReadTsKvQuery query) { if (query.getAggregation() == Aggregation.NONE) { @@ -337,7 +301,7 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD }, MoreExecutors.directExecutor()); } - private long toPartitionTs(long ts) { + long toPartitionTs(long ts) { LocalDateTime time = LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneOffset.UTC); return tsFormat.truncatedTo(time).toInstant(ZoneOffset.UTC).toEpochMilli(); } @@ -417,10 +381,37 @@ public class CassandraBaseTimeseriesDao extends AbstractCassandraBaseTimeseriesD if (isFixedPartitioning()) { //no need to fetch partitions from DB return Futures.immediateFuture(FIXED_PARTITION); } + if (!isUseTsKeyValuePartitioningOnRead()) { + return Futures.immediateFuture(calculatePartitions(minPartition, maxPartition)); + } TbResultSetFuture partitionsFuture = fetchPartitions(tenantId, entityId, query.getKey(), minPartition, maxPartition); return Futures.transformAsync(partitionsFuture, getPartitionsArrayFunction(), readResultsProcessingExecutor); } + List calculatePartitions(long minPartition, long maxPartition) { + if (minPartition == maxPartition) { + return Collections.singletonList(minPartition); + } + List partitions = new ArrayList<>(); + + long currentPartition = minPartition; + LocalDateTime currentPartitionTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(currentPartition), ZoneOffset.UTC); + + while (maxPartition > currentPartition) { + partitions.add(currentPartition); + currentPartitionTime = calculateNextPartition(currentPartitionTime); + currentPartition = currentPartitionTime.toInstant(ZoneOffset.UTC).toEpochMilli(); + } + + partitions.add(maxPartition); + + return partitions; + } + + private LocalDateTime calculateNextPartition(LocalDateTime time) { + return time.plus(1, tsFormat.getTruncateUnit()); + } + private AsyncFunction, List> getFetchChunksAsyncFunction(TenantId tenantId, EntityId entityId, String key, Aggregation aggregation, long startTs, long endTs) { return partitions -> { try { diff --git a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java index 5fd26d400a..4878fdd293 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/timeseries/TimeseriesDao.java @@ -24,7 +24,6 @@ import org.thingsboard.server.common.data.kv.ReadTsKvQueryResult; import org.thingsboard.server.common.data.kv.TsKvEntry; import java.util.List; -import java.util.Map; /** * @author Andrew Shvayka @@ -39,7 +38,5 @@ public interface TimeseriesDao { ListenableFuture remove(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); - ListenableFuture removePartition(TenantId tenantId, EntityId entityId, DeleteTsKvQuery query); - void cleanup(long systemTtl); } diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java index 36eb76edb5..5a4f3af90e 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/NoXssValidatorTest.java @@ -15,23 +15,16 @@ */ package org.thingsboard.server.dao.service; -import org.junit.jupiter.api.BeforeAll; +import com.fasterxml.jackson.databind.node.TextNode; +import org.junit.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.asset.Asset; -import javax.validation.ConstraintValidatorContext; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.mockito.Mockito.mock; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class NoXssValidatorTest { - private static NoXssValidator validator; - - @BeforeAll - public static void beforeAll() { - validator = new NoXssValidator(); - validator.initialize(null); - } @ParameterizedTest @ValueSource(strings = { @@ -44,9 +37,25 @@ public class NoXssValidatorTest { " ", "123 bebe" }) - public void testIsNotValid(String stringWithXss) { - boolean isValid = validator.isValid(stringWithXss, mock(ConstraintValidatorContext.class)); - assertFalse(isValid); + public void givenEntityWithMaliciousPropertyValue_thenReturnValidationError(String maliciousString) { + Asset invalidAsset = new Asset(); + invalidAsset.setName(maliciousString); + + assertThatThrownBy(() -> { + ConstraintValidator.validateFields(invalidAsset); + }).hasMessageContaining("field value is malformed"); + } + + @Test + public void givenEntityWithMaliciousValueInAdditionalInfo_thenReturnValidationError() { + Asset invalidAsset = new Asset(); + String maliciousValue = "qwertyqwerty"; + invalidAsset.setAdditionalInfo(JacksonUtil.newObjectNode() + .set("description", new TextNode(maliciousValue))); + + assertThatThrownBy(() -> { + ConstraintValidator.validateFields(invalidAsset); + }).hasMessageContaining("field value is malformed"); } } diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningDaysAlwaysExistsTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningDaysAlwaysExistsTest.java new file mode 100644 index 0000000000..d5e8350104 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningDaysAlwaysExistsTest.java @@ -0,0 +1,133 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor; + +import java.text.ParseException; +import java.util.List; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = CassandraBaseTimeseriesDao.class) +@TestPropertySource(properties = { + "database.ts.type=cassandra", + "cassandra.query.ts_key_value_partitioning=DAYS", + "cassandra.query.use_ts_key_value_partitioning_on_read=false", + "cassandra.query.ts_key_value_partitions_max_cache_size=100000", + "cassandra.query.ts_key_value_partitions_cache_stats_enabled=true", + "cassandra.query.ts_key_value_partitions_cache_stats_interval=60", + "cassandra.query.ts_key_value_ttl=0", + "cassandra.query.set_null_values_enabled=false", +}) +@Slf4j +public class CassandraBaseTimeseriesDaoPartitioningDaysAlwaysExistsTest { + + @Autowired + CassandraBaseTimeseriesDao tsDao; + + @MockBean(answer = Answers.RETURNS_MOCKS) + @Qualifier("CassandraCluster") + CassandraCluster cassandraCluster; + + @MockBean + CassandraBufferedRateReadExecutor cassandraBufferedRateReadExecutor; + @MockBean + CassandraBufferedRateWriteExecutor cassandraBufferedRateWriteExecutor; + + @Test + public void testToPartitionsDays() throws ParseException { + assertThat(tsDao.getPartitioning()).isEqualTo("DAYS"); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-02T00:00:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-02T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-03T00:00:01Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-03T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T00:00:00Z").getTime()); + } + + @Test + public void testCalculatePartitionsDays() throws ParseException { + long startTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime()); + long nextTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-12T23:59:59Z").getTime()); + long endTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-15T00:00:00Z").getTime()); + log.info("startTs {}, nextTs {}, endTs {}", startTs, nextTs, endTs); + + assertThat(tsDao.calculatePartitions(0, 0)).isEqualTo(List.of(0L)); + assertThat(tsDao.calculatePartitions(0, 1)).isEqualTo(List.of(0L, 1L)); + + assertThat(tsDao.calculatePartitions(startTs, startTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime())); + assertThat(tsDao.calculatePartitions(startTs, nextTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-11T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-12T00:00:00Z").getTime())); + + assertThat(tsDao.calculatePartitions(startTs, endTs)).hasSize(6).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-11T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-12T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-13T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-14T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-15T00:00:00Z").getTime())); + + long leapStartTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-02-27T00:00:00Z").getTime()); + long leapEndTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-03-01T00:00:00Z").getTime()); + assertThat(tsDao.calculatePartitions(leapStartTs, leapEndTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-02-27T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-02-28T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-02-29T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-03-01T00:00:00Z").getTime())); + + long newYearStartTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-12-30T00:00:00Z").getTime()); + long newYearEndTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-01T00:00:00Z").getTime()); + assertThat(tsDao.calculatePartitions(newYearStartTs, newYearEndTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-12-30T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-12-31T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-01T00:00:00Z").getTime())); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningHoursAlwaysExistsTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningHoursAlwaysExistsTest.java new file mode 100644 index 0000000000..d5381653b3 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningHoursAlwaysExistsTest.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor; + +import java.text.ParseException; +import java.util.List; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = CassandraBaseTimeseriesDao.class) +@TestPropertySource(properties = { + "database.ts.type=cassandra", + "cassandra.query.ts_key_value_partitioning=HOURS", + "cassandra.query.use_ts_key_value_partitioning_on_read=false", + "cassandra.query.ts_key_value_partitions_max_cache_size=100000", + "cassandra.query.ts_key_value_partitions_cache_stats_enabled=true", + "cassandra.query.ts_key_value_partitions_cache_stats_interval=60", + "cassandra.query.ts_key_value_ttl=0", + "cassandra.query.set_null_values_enabled=false", +}) +@Slf4j +public class CassandraBaseTimeseriesDaoPartitioningHoursAlwaysExistsTest { + + @Autowired + CassandraBaseTimeseriesDao tsDao; + + @MockBean(answer = Answers.RETURNS_MOCKS) + @Qualifier("CassandraCluster") + CassandraCluster cassandraCluster; + + @MockBean + CassandraBufferedRateReadExecutor cassandraBufferedRateReadExecutor; + @MockBean + CassandraBufferedRateWriteExecutor cassandraBufferedRateWriteExecutor; + + @Test + public void testToPartitionsHours() throws ParseException { + assertThat(tsDao.getPartitioning()).isEqualTo("HOURS"); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-02T01:00:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-02T01:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-03T02:00:01Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-03T02:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T23:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T23:00:00Z").getTime()); + } + + @Test + public void testCalculatePartitionsHours() throws ParseException { + long startTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime()); + long nextTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T03:59:59Z").getTime()); + long endTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-11T00:59:00Z").getTime()); + log.info("startTs {}, nextTs {}, endTs {}", startTs, nextTs, endTs); + + assertThat(tsDao.calculatePartitions(0, 0)).isEqualTo(List.of(0L)); + assertThat(tsDao.calculatePartitions(0, 1)).isEqualTo(List.of(0L, 1L)); + + assertThat(tsDao.calculatePartitions(startTs, startTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime())); + assertThat(tsDao.calculatePartitions(startTs, nextTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T01:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T02:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T03:00:00Z").getTime())); + + assertThat(tsDao.calculatePartitions(startTs, endTs)).hasSize(25).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T01:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T02:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T03:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T04:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T05:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T06:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T07:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T08:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T09:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T10:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T11:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T12:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T13:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T14:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T15:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T16:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T17:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T18:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T19:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T20:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T21:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T22:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T23:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-11T00:00:00Z").getTime())); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningIndefiniteAlwaysExistsTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningIndefiniteAlwaysExistsTest.java new file mode 100644 index 0000000000..82f94cde36 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningIndefiniteAlwaysExistsTest.java @@ -0,0 +1,78 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor; + +import java.text.ParseException; +import java.util.List; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = CassandraBaseTimeseriesDao.class) +@TestPropertySource(properties = { + "database.ts.type=cassandra", + "cassandra.query.ts_key_value_partitioning=INDEFINITE", + "cassandra.query.use_ts_key_value_partitioning_on_read=false", + "cassandra.query.ts_key_value_partitions_max_cache_size=100000", + "cassandra.query.ts_key_value_partitions_cache_stats_enabled=true", + "cassandra.query.ts_key_value_partitions_cache_stats_interval=60", + "cassandra.query.ts_key_value_ttl=0", + "cassandra.query.set_null_values_enabled=false", +}) +@Slf4j +public class CassandraBaseTimeseriesDaoPartitioningIndefiniteAlwaysExistsTest { + + @Autowired + CassandraBaseTimeseriesDao tsDao; + + @MockBean(answer = Answers.RETURNS_MOCKS) + @Qualifier("CassandraCluster") + CassandraCluster cassandraCluster; + + @MockBean + CassandraBufferedRateReadExecutor cassandraBufferedRateReadExecutor; + @MockBean + CassandraBufferedRateWriteExecutor cassandraBufferedRateWriteExecutor; + + @Test + public void testToPartitionsIndefinite() throws ParseException { + assertThat(tsDao.getPartitioning()).isEqualTo("INDEFINITE"); + assertThat(tsDao.toPartitionTs(ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime())).isEqualTo(0L); + } + + + @Test + public void testCalculatePartitionsIndefinite() throws ParseException { + //Indefinite partitioning should never call tsDao.calculatePartitions() + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMinutesAlwaysExistsTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMinutesAlwaysExistsTest.java new file mode 100644 index 0000000000..ffdf18c42b --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMinutesAlwaysExistsTest.java @@ -0,0 +1,121 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor; + +import java.text.ParseException; +import java.util.List; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = CassandraBaseTimeseriesDao.class) +@TestPropertySource(properties = { + "database.ts.type=cassandra", + "cassandra.query.ts_key_value_partitioning=MINUTES", + "cassandra.query.use_ts_key_value_partitioning_on_read=false", + "cassandra.query.ts_key_value_partitions_max_cache_size=100000", + "cassandra.query.ts_key_value_partitions_cache_stats_enabled=true", + "cassandra.query.ts_key_value_partitions_cache_stats_interval=60", + "cassandra.query.ts_key_value_ttl=0", + "cassandra.query.set_null_values_enabled=false", +}) +@Slf4j +public class CassandraBaseTimeseriesDaoPartitioningMinutesAlwaysExistsTest { + + @Autowired + CassandraBaseTimeseriesDao tsDao; + + @MockBean(answer = Answers.RETURNS_MOCKS) + @Qualifier("CassandraCluster") + CassandraCluster cassandraCluster; + + @MockBean + CassandraBufferedRateReadExecutor cassandraBufferedRateReadExecutor; + @MockBean + CassandraBufferedRateWriteExecutor cassandraBufferedRateWriteExecutor; + + @Test + public void testToPartitionsMinutes() throws ParseException { + assertThat(tsDao.getPartitioning()).isEqualTo("MINUTES"); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-02T00:01:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-02T00:01:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-03T00:02:01Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-03T00:02:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T23:59:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T23:59:00Z").getTime()); + } + + + @Test + public void testCalculatePartitionsMinutes() throws ParseException { + long startTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime()); + long nextTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:02:59Z").getTime()); + long endTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:10:00Z").getTime()); + log.info("startTs {}, nextTs {}, endTs {}", startTs, nextTs, endTs); + + assertThat(tsDao.calculatePartitions(0, 0)).isEqualTo(List.of(0L)); + assertThat(tsDao.calculatePartitions(0, 1)).isEqualTo(List.of(0L, 1L)); + + assertThat(tsDao.calculatePartitions(startTs, startTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime())); + assertThat(tsDao.calculatePartitions(startTs, nextTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:01:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:02:00Z").getTime())); + + assertThat(tsDao.calculatePartitions(startTs, endTs)).hasSize(11).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:01:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:02:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:03:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:04:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:05:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:06:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:07:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:08:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:09:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-10-10T00:10:00Z").getTime())); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest.java new file mode 100644 index 0000000000..8c7d12da86 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest.java @@ -0,0 +1,134 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor; + +import java.text.ParseException; +import java.util.List; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = CassandraBaseTimeseriesDao.class) +@TestPropertySource(properties = { + "database.ts.type=cassandra", + "cassandra.query.ts_key_value_partitioning=MONTHS", + "cassandra.query.use_ts_key_value_partitioning_on_read=false", + "cassandra.query.ts_key_value_partitions_max_cache_size=100000", + "cassandra.query.ts_key_value_partitions_cache_stats_enabled=true", + "cassandra.query.ts_key_value_partitions_cache_stats_interval=60", + "cassandra.query.ts_key_value_ttl=0", + "cassandra.query.set_null_values_enabled=false", +}) +@Slf4j +public class CassandraBaseTimeseriesDaoPartitioningMonthsAlwaysExistsTest { + + @Autowired + CassandraBaseTimeseriesDao tsDao; + + @MockBean(answer = Answers.RETURNS_MOCKS) + @Qualifier("CassandraCluster") + CassandraCluster cassandraCluster; + + @MockBean + CassandraBufferedRateReadExecutor cassandraBufferedRateReadExecutor; + @MockBean + CassandraBufferedRateWriteExecutor cassandraBufferedRateWriteExecutor; + + @Test + public void testToPartitionsMonths() throws ParseException { + assertThat(tsDao.getPartitioning()).isEqualTo("MONTHS"); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime())).isEqualTo(1640995200000L).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-01T00:00:00Z").getTime())).isEqualTo(1651363200000L).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-01T00:00:01Z").getTime())).isEqualTo(1651363200000L).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T23:59:59Z").getTime())).isEqualTo(1651363200000L).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T23:59:59Z").getTime())).isEqualTo(1701388800000L).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-01T00:00:00Z").getTime()); + } + + @Test + public void testCalculatePartitionsMonths() throws ParseException { + long startTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-12T00:00:00Z").getTime()); + long nextTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-01-31T23:59:59Z").getTime()); + long leapTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-02-29T23:59:59Z").getTime()); + long endTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-31T23:59:59Z").getTime()); + log.info("startTs {}, nextTs {}, leapTs {}, endTs {}", startTs, nextTs, leapTs, endTs); + + assertThat(tsDao.calculatePartitions(0, 0)).isEqualTo(List.of(0L)); + assertThat(tsDao.calculatePartitions(0, 1)).isEqualTo(List.of(0L, 1L)); + + assertThat(tsDao.calculatePartitions(startTs, startTs)).isEqualTo(List.of(1575158400000L)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-01T00:00:00Z").getTime())); + assertThat(tsDao.calculatePartitions(startTs, nextTs)).isEqualTo(List.of(1575158400000L, 1577836800000L)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-01-01T00:00:00Z").getTime())); + + assertThat(tsDao.calculatePartitions(startTs, leapTs)).isEqualTo(List.of(1575158400000L, 1577836800000L, 1580515200000L)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-02-01T00:00:00Z").getTime())); + + assertThat(tsDao.calculatePartitions(startTs, endTs)).hasSize(14).isEqualTo(List.of( + 1575158400000L, + 1577836800000L, 1580515200000L, 1583020800000L, + 1585699200000L, 1588291200000L, 1590969600000L, + 1593561600000L, 1596240000000L, 1598918400000L, + 1601510400000L, 1604188800000L, 1606780800000L, + 1609459200000L)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-12-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-02-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-03-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-04-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-05-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-06-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-07-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-08-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-09-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-10-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-11-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-12-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-01T00:00:00Z").getTime())); + } + +} diff --git a/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningYearsAlwaysExistsTest.java b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningYearsAlwaysExistsTest.java new file mode 100644 index 0000000000..4b2bcb6455 --- /dev/null +++ b/dao/src/test/java/org/thingsboard/server/dao/timeseries/CassandraBaseTimeseriesDaoPartitioningYearsAlwaysExistsTest.java @@ -0,0 +1,116 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.dao.timeseries; + +import lombok.extern.slf4j.Slf4j; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; +import org.thingsboard.server.dao.cassandra.CassandraCluster; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateReadExecutor; +import org.thingsboard.server.dao.nosql.CassandraBufferedRateWriteExecutor; + +import java.text.ParseException; +import java.util.List; + +import static org.apache.commons.lang3.time.DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(SpringRunner.class) +@SpringBootTest(classes = CassandraBaseTimeseriesDao.class) +@TestPropertySource(properties = { + "database.ts.type=cassandra", + "cassandra.query.ts_key_value_partitioning=YEARS", + "cassandra.query.use_ts_key_value_partitioning_on_read=false", + "cassandra.query.ts_key_value_partitions_max_cache_size=100000", + "cassandra.query.ts_key_value_partitions_cache_stats_enabled=true", + "cassandra.query.ts_key_value_partitions_cache_stats_interval=60", + "cassandra.query.ts_key_value_ttl=0", + "cassandra.query.set_null_values_enabled=false", +}) +@Slf4j +public class CassandraBaseTimeseriesDaoPartitioningYearsAlwaysExistsTest { + + @Autowired + CassandraBaseTimeseriesDao tsDao; + + @MockBean(answer = Answers.RETURNS_MOCKS) + @Qualifier("CassandraCluster") + CassandraCluster cassandraCluster; + + @MockBean + CassandraBufferedRateReadExecutor cassandraBufferedRateReadExecutor; + @MockBean + CassandraBufferedRateWriteExecutor cassandraBufferedRateWriteExecutor; + + @Test + public void testToPartitionsYears() throws ParseException { + assertThat(tsDao.getPartitioning()).isEqualTo("YEARS"); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-01T00:00:00Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-01T00:00:01Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-05-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime()); + assertThat(tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-12-31T23:59:59Z").getTime())).isEqualTo( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-01-01T00:00:00Z").getTime()); + } + + @Test + public void testCalculatePartitionsYears() throws ParseException { + long startTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-01-01T00:00:00Z").getTime()); + long nextTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2021-10-12T23:59:59Z").getTime()); + long endTs = tsDao.toPartitionTs( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2025-07-15T00:00:00Z").getTime()); + log.info("startTs {}, nextTs {}, endTs {}", startTs, nextTs, endTs); + + assertThat(tsDao.calculatePartitions(0, 0)).isEqualTo(List.of(0L)); + assertThat(tsDao.calculatePartitions(0, 1)).isEqualTo(List.of(0L, 1L)); + + assertThat(tsDao.calculatePartitions(startTs, startTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-01-01T00:00:00Z").getTime())); + assertThat(tsDao.calculatePartitions(startTs, nextTs)).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-01T00:00:00Z").getTime())); + + assertThat(tsDao.calculatePartitions(startTs, endTs)).hasSize(7).isEqualTo(List.of( + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2019-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2020-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2021-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2022-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2023-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2024-01-01T00:00:00Z").getTime(), + ISO_DATETIME_TIME_ZONE_FORMAT.parse("2025-01-01T00:00:00Z").getTime())); + } + +} diff --git a/docker/tb-js-executor.env b/docker/tb-js-executor.env index e080906549..1938449d53 100644 --- a/docker/tb-js-executor.env +++ b/docker/tb-js-executor.env @@ -3,4 +3,5 @@ LOGGER_LEVEL=info LOG_FOLDER=logs LOGGER_FILENAME=tb-js-executor-%DATE%.log DOCKER_MODE=true -SCRIPT_BODY_TRACE_FREQUENCY=1000 \ No newline at end of file +SCRIPT_BODY_TRACE_FREQUENCY=1000 +NODE_OPTIONS="--max-old-space-size=200" diff --git a/lombok.config b/lombok.config index d904701090..1b8f891cd9 100644 --- a/lombok.config +++ b/lombok.config @@ -1,2 +1,3 @@ config.stopbubbling = true lombok.anyconstructor.addconstructorproperties = true +lombok.copyableAnnotations += org.springframework.context.annotation.Lazy diff --git a/msa/black-box-tests/README.md b/msa/black-box-tests/README.md index a45badf44a..a60e7405a8 100644 --- a/msa/black-box-tests/README.md +++ b/msa/black-box-tests/README.md @@ -30,5 +30,9 @@ As result, in REPOSITORY column, next images should be present: mvn clean install -DblackBoxTests.skip=false -DblackBoxTests.hybridMode=true +To run the black box tests with using local env run tests in the [msa/black-box-tests](../black-box-tests) directory with runLocal property: + + mvn clean install -DblackBoxTests.skip=false -DrunLocal=true + diff --git a/msa/black-box-tests/pom.xml b/msa/black-box-tests/pom.xml index a1c745e163..49f4c959e9 100644 --- a/msa/black-box-tests/pom.xml +++ b/msa/black-box-tests/pom.xml @@ -57,11 +57,6 @@ httpclient test - - io.takari.junit - takari-cpsuite - test - org.springframework.boot spring-boot-starter-test @@ -72,6 +67,26 @@ junit-vintage-engine test + + org.testng + testng + test + + + org.assertj + assertj-core + test + + + io.rest-assured + rest-assured + test + + + org.hamcrest + hamcrest-all + test + org.awaitility awaitility @@ -152,11 +167,18 @@ org.apache.maven.plugins maven-surefire-plugin - - **/*TestSuite.java - + + src/test/resources/testNG.xml + ${blackBoxTests.skip} + + + org.apache.maven.surefire + surefire-testng + ${surefire.version} + + diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java index 07529baead..a148d7d138 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/AbstractContainerTest.java @@ -15,115 +15,59 @@ */ package org.thingsboard.server.msa; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.gson.JsonArray; import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import lombok.extern.slf4j.Slf4j; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.socket.ConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.TrustStrategy; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.SSLContexts; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.rules.TestRule; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; -import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; -import org.thingsboard.rest.client.RestClient; -import org.thingsboard.server.common.data.Device; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.Listeners; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; -import org.thingsboard.server.msa.mapper.WsTelemetryResponse; -import javax.net.ssl.SSLContext; import java.net.URI; -import java.util.List; import java.util.Map; import java.util.Random; + @Slf4j +@Listeners(TestListener.class) public abstract class AbstractContainerTest { - protected static final String HTTPS_URL = "https://localhost"; - protected static final String WSS_URL = "wss://localhost"; - protected static String TB_TOKEN; - protected static RestClient restClient; - protected static long timeoutMultiplier = 1; - protected ObjectMapper mapper = new ObjectMapper(); - protected JsonParser jsonParser = new JsonParser(); - - @BeforeClass - public static void before() throws Exception { - restClient = new RestClient(HTTPS_URL); - restClient.getRestTemplate().setRequestFactory(getRequestFactoryForSelfSignedCert()); + private static final ContainerTestSuite containerTestSuite = ContainerTestSuite.getInstance(); + protected static TestRestClient testRestClient; + @BeforeSuite + public void beforeSuite() { + if ("false".equals(System.getProperty("runLocal", "false"))) { + containerTestSuite.start(); + } + testRestClient = new TestRestClient(TestProperties.getBaseUrl()); if (!"kafka".equals(System.getProperty("blackBoxTests.queue", "kafka"))) { timeoutMultiplier = 10; } } - @Rule - public TestRule watcher = new TestWatcher() { - protected void starting(Description description) { - log.info("================================================="); - log.info("STARTING TEST: {}" , description.getMethodName()); - log.info("================================================="); + @AfterSuite + public void afterSuite() { + if (containerTestSuite.isActive()) { + containerTestSuite.stop(); } - - /** - * Invoked when a test succeeds - */ - protected void succeeded(Description description) { - log.info("================================================="); - log.info("SUCCEEDED TEST: {}" , description.getMethodName()); - log.info("================================================="); - } - - /** - * Invoked when a test fails - */ - protected void failed(Throwable e, Description description) { - log.info("================================================="); - log.info("FAILED TEST: {}" , description.getMethodName(), e); - log.info("================================================="); - } - }; - - protected Device createGatewayDevice() throws JsonProcessingException { - String isGateway = "{\"gateway\":true}"; - ObjectMapper objectMapper = new ObjectMapper(); - JsonNode additionalInfo = objectMapper.readTree(isGateway); - Device gatewayDeviceTemplate = new Device(); - gatewayDeviceTemplate.setName("mqtt_gateway"); - gatewayDeviceTemplate.setType("gateway"); - gatewayDeviceTemplate.setAdditionalInfo(additionalInfo); - return restClient.saveDevice(gatewayDeviceTemplate); - } - - protected Device createDevice(String name) { - Device device = new Device(); - device.setName(name + StringUtils.randomAlphanumeric(7)); - device.setType("DEFAULT"); - return restClient.saveDevice(device); } protected WsClient subscribeToWebSocket(DeviceId deviceId, String scope, CmdsType property) throws Exception { - WsClient wsClient = new WsClient(new URI(WSS_URL + "/api/ws/plugins/telemetry?token=" + restClient.getToken()), timeoutMultiplier); - SSLContextBuilder builder = SSLContexts.custom(); - builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true); - wsClient.setSocketFactory(builder.build().getSocketFactory()); + String webSocketUrl = TestProperties.getWebSocketUrl(); + WsClient wsClient = new WsClient(new URI(webSocketUrl + "/api/ws/plugins/telemetry?token=" + testRestClient.getToken()), timeoutMultiplier); + if (webSocketUrl.matches("^(wss)://.*$")) { + SSLContextBuilder builder = SSLContexts.custom(); + builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true); + wsClient.setSocketFactory(builder.build().getSocketFactory()); + } wsClient.connectBlocking(); JsonObject cmdsObject = new JsonObject(); @@ -150,16 +94,6 @@ public abstract class AbstractContainerTest { .build(); } - protected boolean verify(WsTelemetryResponse wsTelemetryResponse, String key, Long expectedTs, String expectedValue) { - List list = wsTelemetryResponse.getDataValuesByKey(key); - return expectedTs.equals(list.get(0)) && expectedValue.equals(list.get(1)); - } - - protected boolean verify(WsTelemetryResponse wsTelemetryResponse, String key, String expectedValue) { - List list = wsTelemetryResponse.getDataValuesByKey(key); - return expectedValue.equals(list.get(1)); - } - protected JsonObject createGatewayConnectPayload(String deviceName){ JsonObject payload = new JsonObject(); payload.addProperty("device", deviceName); @@ -216,20 +150,4 @@ public abstract class AbstractContainerTest { } } - private static HttpComponentsClientHttpRequestFactory getRequestFactoryForSelfSignedCert() throws Exception { - SSLContextBuilder builder = SSLContexts.custom(); - builder.loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true); - SSLContext sslContext = builder.build(); - SSLConnectionSocketFactory sslSelfSigned = new SSLConnectionSocketFactory(sslContext, (s, sslSession) -> true); - - Registry socketFactoryRegistry = RegistryBuilder - .create() - .register("https", sslSelfSigned) - .build(); - - PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(socketFactoryRegistry); - CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build(); - return new HttpComponentsClientHttpRequestFactory(httpClient); - } - } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java index 47acbde16e..2ff7088fa0 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ContainerTestSuite.java @@ -17,9 +17,6 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.junit.ClassRule; -import org.junit.extensions.cpsuite.ClasspathSuite; -import org.junit.runner.RunWith; import org.testcontainers.containers.DockerComposeContainer; import org.testcontainers.containers.wait.strategy.Wait; import org.thingsboard.server.common.data.StringUtils; @@ -41,10 +38,8 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.Assert.fail; +import static org.testng.Assert.fail; -@RunWith(ClasspathSuite.class) -@ClasspathSuite.ClassnameFilters({"org.thingsboard.server.msa.*Test"}) @Slf4j public class ContainerTestSuite { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); @@ -57,107 +52,133 @@ public class ContainerTestSuite { private static final String TB_JS_EXECUTOR_LOG_REGEXP = ".*template started.*"; private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(400); - private static DockerComposeContainer testContainer; - - @ClassRule - public static ThingsBoardDbInstaller installTb = new ThingsBoardDbInstaller(); - - @ClassRule - public static DockerComposeContainer getTestContainer() { - if (testContainer == null) { - log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); - log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); - boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); - try { - final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; - log.info("targetDir {}", targetDir); - FileUtils.copyDirectory(new File(SOURCE_DIR), new File(targetDir)); - replaceInFile(targetDir + "docker-compose.yml", " container_name: \"${LOAD_BALANCER_NAME}\"", "", "container_name"); - - FileUtils.copyDirectory(new File("src/test/resources"), new File(targetDir)); - - class DockerComposeContainerImpl> extends DockerComposeContainer { - public DockerComposeContainerImpl(List composeFiles) { - super(composeFiles); - } - - @Override - public void stop() { - super.stop(); - tryDeleteDir(targetDir); - } - } + private DockerComposeContainer testContainer; + private ThingsBoardDbInstaller installTb; + private boolean isActive; + + private static ContainerTestSuite containerTestSuite; + + public boolean isActive() { + return isActive; + } + + public void setActive(boolean active) { + isActive = active; + } + + private ContainerTestSuite() { + } + + public static ContainerTestSuite getInstance() { + if (containerTestSuite == null) { + containerTestSuite = new ContainerTestSuite(); + } + return containerTestSuite; + } + + public void start() { + installTb = new ThingsBoardDbInstaller(); + installTb.createVolumes(); + log.info("System property of blackBoxTests.redisCluster is {}", IS_REDIS_CLUSTER); + log.info("System property of blackBoxTests.hybridMode is {}", IS_HYBRID_MODE); + boolean skipTailChildContainers = Boolean.valueOf(System.getProperty("blackBoxTests.skipTailChildContainers")); + try { + final String targetDir = FileUtils.getTempDirectoryPath() + "/" + "ContainerTestSuite-" + UUID.randomUUID() + "/"; + log.info("targetDir {}", targetDir); + FileUtils.copyDirectory(new File(SOURCE_DIR), new File(targetDir)); + replaceInFile(targetDir + "docker-compose.yml", " container_name: \"${LOAD_BALANCER_NAME}\"", "", "container_name"); - List composeFiles = new ArrayList<>(Arrays.asList( - new File(targetDir + "docker-compose.yml"), - new File(targetDir + "docker-compose.volumes.yml"), - new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), - new File(targetDir + "docker-compose.postgres.volumes.yml"), - new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), - new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), - new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")) - )); - - Map queueEnv = new HashMap<>(); - queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE); - switch (QUEUE_TYPE) { - case "kafka": - composeFiles.add(new File(targetDir + "docker-compose.kafka.yml")); - break; - case "aws-sqs": - replaceInFile(targetDir, "queue-aws-sqs.env", - Map.of("YOUR_KEY", getSysProp("blackBoxTests.awsKey"), - "YOUR_SECRET", getSysProp("blackBoxTests.awsSecret"), - "YOUR_REGION", getSysProp("blackBoxTests.awsRegion"))); - break; - case "rabbitmq": - composeFiles.add(new File(targetDir + "docker-compose.rabbitmq-server.yml")); - replaceInFile(targetDir, "queue-rabbitmq.env", - Map.of("localhost", "rabbitmq")); - break; - case "service-bus": - replaceInFile(targetDir, "queue-service-bus.env", - Map.of("YOUR_NAMESPACE_NAME", getSysProp("blackBoxTests.serviceBusNamespace"), - "YOUR_SAS_KEY_NAME", getSysProp("blackBoxTests.serviceBusSASPolicy"))); - replaceInFile(targetDir, "queue-service-bus.env", - Map.of("YOUR_SAS_KEY", getSysProp("blackBoxTests.serviceBusPrimaryKey"))); - break; - case "pubsub": - replaceInFile(targetDir, "queue-pubsub.env", - Map.of("YOUR_PROJECT_ID", getSysProp("blackBoxTests.pubSubProjectId"), - "YOUR_SERVICE_ACCOUNT", getSysProp("blackBoxTests.pubSubServiceAccount"))); - break; - default: - throw new RuntimeException("Unsupported queue type: " + QUEUE_TYPE); + FileUtils.copyDirectory(new File("src/test/resources"), new File(targetDir)); + + class DockerComposeContainerImpl> extends DockerComposeContainer { + public DockerComposeContainerImpl(List composeFiles) { + super(composeFiles); } - if (IS_HYBRID_MODE) { - composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); + @Override + public void stop() { + super.stop(); + tryDeleteDir(targetDir); } + } - testContainer = new DockerComposeContainerImpl<>(composeFiles) - .withPull(false) - .withLocalCompose(true) - .withTailChildContainers(!skipTailChildContainers) - .withEnv(installTb.getEnv()) - .withEnv(queueEnv) - .withEnv("LOAD_BALANCER_NAME", "") - .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) - .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); - } catch (Exception e) { - log.error("Failed to create test container", e); - fail("Failed to create test container"); + List composeFiles = new ArrayList<>(Arrays.asList( + new File(targetDir + "docker-compose.yml"), + new File(targetDir + "docker-compose.volumes.yml"), + new File(targetDir + (IS_HYBRID_MODE ? "docker-compose.hybrid.yml" : "docker-compose.postgres.yml")), + new File(targetDir + "docker-compose.postgres.volumes.yml"), + new File(targetDir + "docker-compose." + QUEUE_TYPE + ".yml"), + new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.yml" : "docker-compose.redis.yml")), + new File(targetDir + (IS_REDIS_CLUSTER ? "docker-compose.redis-cluster.volumes.yml" : "docker-compose.redis.volumes.yml")) + )); + + Map queueEnv = new HashMap<>(); + queueEnv.put("TB_QUEUE_TYPE", QUEUE_TYPE); + switch (QUEUE_TYPE) { + case "kafka": + composeFiles.add(new File(targetDir + "docker-compose.kafka.yml")); + break; + case "aws-sqs": + replaceInFile(targetDir, "queue-aws-sqs.env", + Map.of("YOUR_KEY", getSysProp("blackBoxTests.awsKey"), + "YOUR_SECRET", getSysProp("blackBoxTests.awsSecret"), + "YOUR_REGION", getSysProp("blackBoxTests.awsRegion"))); + break; + case "rabbitmq": + composeFiles.add(new File(targetDir + "docker-compose.rabbitmq-server.yml")); + replaceInFile(targetDir, "queue-rabbitmq.env", + Map.of("localhost", "rabbitmq")); + break; + case "service-bus": + replaceInFile(targetDir, "queue-service-bus.env", + Map.of("YOUR_NAMESPACE_NAME", getSysProp("blackBoxTests.serviceBusNamespace"), + "YOUR_SAS_KEY_NAME", getSysProp("blackBoxTests.serviceBusSASPolicy"))); + replaceInFile(targetDir, "queue-service-bus.env", + Map.of("YOUR_SAS_KEY", getSysProp("blackBoxTests.serviceBusPrimaryKey"))); + break; + case "pubsub": + replaceInFile(targetDir, "queue-pubsub.env", + Map.of("YOUR_PROJECT_ID", getSysProp("blackBoxTests.pubSubProjectId"), + "YOUR_SERVICE_ACCOUNT", getSysProp("blackBoxTests.pubSubServiceAccount"))); + break; + default: + throw new RuntimeException("Unsupported queue type: " + QUEUE_TYPE); } + + if (IS_HYBRID_MODE) { + composeFiles.add(new File(targetDir + "docker-compose.cassandra.volumes.yml")); + } + + testContainer = new DockerComposeContainerImpl<>(composeFiles) + .withPull(false) + .withLocalCompose(true) + .withTailChildContainers(!skipTailChildContainers) + .withEnv(installTb.getEnv()) + .withEnv(queueEnv) + .withEnv("LOAD_BALANCER_NAME", "") + .withExposedService("haproxy", 80, Wait.forHttp("/swagger-ui.html").withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-core1", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-core2", Wait.forLogMessage(TB_CORE_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-http-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-http-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-mqtt-transport1", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-mqtt-transport2", Wait.forLogMessage(TRANSPORTS_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-vc-executor1", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-vc-executor2", Wait.forLogMessage(TB_VC_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)) + .waitingFor("tb-js-executor", Wait.forLogMessage(TB_JS_EXECUTOR_LOG_REGEXP, 1).withStartupTimeout(CONTAINER_STARTUP_TIMEOUT)); + testContainer.start(); + setActive(true); + } catch (Exception e) { + log.error("Failed to create test container", e); + fail("Failed to create test container"); + } + } + public void stop() { + if (isActive) { + testContainer.stop(); + installTb.savaLogsAndRemoveVolumes(); + setActive(false); } - return testContainer; } private static void replaceInFile(String targetDir, String fileName, Map replacements) throws IOException { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestListener.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestListener.java new file mode 100644 index 0000000000..51bc75c86a --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestListener.java @@ -0,0 +1,53 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import lombok.extern.slf4j.Slf4j; +import org.testng.ITestContext; +import org.testng.ITestResult; +import org.testng.TestListenerAdapter; + +import static org.testng.internal.Utils.log; + +@Slf4j +public class TestListener extends TestListenerAdapter { + + @Override + public void onTestStart(ITestResult result) { + super.onTestStart(result); + log.info("===>>> Test started: " + result.getName()); + } + + /** + * Invoked when a test succeeds + */ + @Override + public void onTestSuccess(ITestResult result) { + super.onTestSuccess(result); + if (result != null) { + log.info("<<<=== Test completed successfully: " + result.getName()); + } + } + + /** + * Invoked when a test fails + */ + @Override + public void onTestFailure(ITestResult result) { + super.onTestFailure(result); + log.info("<<<=== Test failed: " + result.getName()); + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java new file mode 100644 index 0000000000..020dbf8b93 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestProperties.java @@ -0,0 +1,61 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +@Slf4j +public class TestProperties { + + private static final String HTTPS_URL = "https://localhost"; + + private static final String WSS_URL = "wss://localhost"; + + private static final ContainerTestSuite instance = ContainerTestSuite.getInstance(); + + private static Properties properties; + + public static String getBaseUrl() { + if (instance.isActive()) { + return HTTPS_URL; + } + return getProperties().getProperty("tb.baseUrl"); + } + + public static String getWebSocketUrl() { + if (instance.isActive()) { + return WSS_URL; + } + return getProperties().getProperty("tb.wsUrl"); + } + + private static Properties getProperties() { + if (properties == null) { + try (InputStream input = TestProperties.class.getClassLoader().getResourceAsStream("config.properties")) { + properties = new Properties(); + properties.load(input); + } catch (IOException ex) { + log.error("Exception while reading test properties " + ex.getMessage()); + } + } + return properties; + } + +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java new file mode 100644 index 0000000000..713205e228 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/TestRestClient.java @@ -0,0 +1,262 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa; + +import com.fasterxml.jackson.databind.JsonNode; +import io.restassured.RestAssured; +import io.restassured.common.mapper.TypeRef; +import io.restassured.config.HeaderConfig; +import io.restassured.config.RestAssuredConfig; +import io.restassured.filter.log.RequestLoggingFilter; +import io.restassured.filter.log.ResponseLoggingFilter; +import io.restassured.http.ContentType; +import io.restassured.path.json.JsonPath; +import io.restassured.response.ValidatableResponse; +import io.restassured.specification.RequestSpecification; +import org.thingsboard.server.common.data.Device; +import org.thingsboard.server.common.data.id.DeviceId; +import org.thingsboard.server.common.data.id.EntityId; +import org.thingsboard.server.common.data.id.RuleChainId; +import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; +import org.thingsboard.server.common.data.relation.EntityRelation; +import org.thingsboard.server.common.data.relation.RelationTypeGroup; +import org.thingsboard.server.common.data.rule.RuleChain; +import org.thingsboard.server.common.data.rule.RuleChainMetaData; +import org.thingsboard.server.common.data.security.DeviceCredentials; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static io.restassured.RestAssured.given; +import static java.net.HttpURLConnection.HTTP_NOT_FOUND; +import static java.net.HttpURLConnection.HTTP_OK; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.core.AnyOf.anyOf; +import static org.thingsboard.server.common.data.StringUtils.isEmpty; + +public class TestRestClient { + private static final String JWT_TOKEN_HEADER_PARAM = "X-Authorization"; + private static final String CONTENT_TYPE_HEADER = "Content-Type"; + private final RequestSpecification requestSpec; + private String token; + private String refreshToken; + + public TestRestClient(String url) { + RestAssured.filters(new RequestLoggingFilter(), new ResponseLoggingFilter()); + + requestSpec = given().baseUri(url) + .contentType(ContentType.JSON) + .config(RestAssuredConfig.config() + .headerConfig(HeaderConfig.headerConfig() + .overwriteHeadersWithName(JWT_TOKEN_HEADER_PARAM, CONTENT_TYPE_HEADER))); + + if (url.matches("^(https)://.*$")) { + requestSpec.relaxedHTTPSValidation(); + } + } + + public void login(String username, String password) { + Map loginRequest = new HashMap<>(); + loginRequest.put("username", username); + loginRequest.put("password", password); + + JsonPath jsonPath = given().spec(requestSpec).body(loginRequest) + .post( "/api/auth/login") + .getBody().jsonPath(); + token = jsonPath.get("token"); + refreshToken = jsonPath.get("refreshToken"); + requestSpec.header(JWT_TOKEN_HEADER_PARAM, "Bearer " + token); + } + + public Device postDevice(String accessToken, Device device) { + return given().spec(requestSpec).body(device) + .pathParams("accessToken", accessToken) + .post("/api/device?accessToken={accessToken}") + .then() + .statusCode(HTTP_OK) + .extract() + .as(Device.class); + } + + public ValidatableResponse getDeviceById(DeviceId deviceId, int statusCode) { + return given().spec(requestSpec) + .pathParams("deviceId", deviceId.getId()) + .get("/api/device/{deviceId}") + .then() + .statusCode(statusCode); + } + public Device getDeviceById(DeviceId deviceId) { + return getDeviceById(deviceId, HTTP_OK) + .extract() + .as(Device.class); + } + public DeviceCredentials getDeviceCredentialsByDeviceId(DeviceId deviceId) { + return given().spec(requestSpec).get("/api/device/{deviceId}/credentials", deviceId.getId()) + .then() + .assertThat() + .statusCode(HTTP_OK) + .extract() + .as(DeviceCredentials.class); + } + + public ValidatableResponse postTelemetry(String credentialsId, JsonNode telemetry) { + return given().spec(requestSpec).body(telemetry) + .post("/api/v1/{credentialsId}/telemetry", credentialsId) + .then() + .statusCode(HTTP_OK); + } + + public ValidatableResponse deleteDevice(DeviceId deviceId) { + return given().spec(requestSpec) + .delete("/api/device/{deviceId}", deviceId.getId()) + .then() + .statusCode(HTTP_OK); + } + public ValidatableResponse deleteDeviceIfExists(DeviceId deviceId) { + return given().spec(requestSpec) + .delete("/api/device/{deviceId}", deviceId.getId()) + .then() + .statusCode(anyOf(is(HTTP_OK),is(HTTP_NOT_FOUND))); + } + + public ValidatableResponse postTelemetryAttribute(String entityType, DeviceId deviceId, String scope, JsonNode attribute) { + return given().spec(requestSpec).body(attribute) + .post("/api/plugins/telemetry/{entityType}/{entityId}/attributes/{scope}", entityType, deviceId.getId(), scope) + .then() + .statusCode(HTTP_OK); + } + + public ValidatableResponse postAttribute(String accessToken, JsonNode attribute) { + return given().spec(requestSpec).body(attribute) + .post("/api/v1/{accessToken}/attributes/", accessToken) + .then() + .statusCode(HTTP_OK); + } + + public JsonNode getAttributes(String accessToken, String clientKeys, String sharedKeys) { + return given().spec(requestSpec) + .queryParam("clientKeys", clientKeys) + .queryParam("sharedKeys", sharedKeys) + .get("/api/v1/{accessToken}/attributes", accessToken) + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + + public PageData getRuleChains(PageLink pageLink) { + Map params = new HashMap<>(); + addPageLinkToParam(params, pageLink); + return given().spec(requestSpec).queryParams(params) + .get("/api/ruleChains") + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef>() {}); + } + + public RuleChain postRootRuleChain(RuleChain ruleChain) { + return given().spec(requestSpec) + .body(ruleChain) + .post("/api/ruleChain") + .then() + .statusCode(HTTP_OK) + .extract() + .as(RuleChain.class); + } + + public RuleChainMetaData postRuleChainMetadata(RuleChainMetaData ruleChainMetaData) { + return given().spec(requestSpec) + .body(ruleChainMetaData) + .post("/api/ruleChain/metadata") + .then() + .statusCode(HTTP_OK) + .extract() + .as(RuleChainMetaData.class); + } + + public void setRootRuleChain(RuleChainId ruleChainId) { + given().spec(requestSpec) + .post("/api/ruleChain/{ruleChainId}/root", ruleChainId.getId()) + .then() + .statusCode(HTTP_OK); + } + + public void deleteRuleChain(RuleChainId ruleChainId) { + given().spec(requestSpec) + .delete("/api/ruleChain/{ruleChainId}", ruleChainId.getId()) + .then() + .statusCode(HTTP_OK); + } + + private String getUrlParams(PageLink pageLink) { + String urlParams = "pageSize={pageSize}&page={page}"; + if (!isEmpty(pageLink.getTextSearch())) { + urlParams += "&textSearch={textSearch}"; + } + if (pageLink.getSortOrder() != null) { + urlParams += "&sortProperty={sortProperty}&sortOrder={sortOrder}"; + } + return urlParams; + } + + private void addPageLinkToParam(Map params, PageLink pageLink) { + params.put("pageSize", String.valueOf(pageLink.getPageSize())); + params.put("page", String.valueOf(pageLink.getPage())); + if (!isEmpty(pageLink.getTextSearch())) { + params.put("textSearch", pageLink.getTextSearch()); + } + if (pageLink.getSortOrder() != null) { + params.put("sortProperty", pageLink.getSortOrder().getProperty()); + params.put("sortOrder", pageLink.getSortOrder().getDirection().name()); + } + } + + public List findRelationByFrom(EntityId fromId, RelationTypeGroup relationTypeGroup) { + Map params = new HashMap<>(); + params.put("fromId", fromId.getId().toString()); + params.put("fromType", fromId.getEntityType().name()); + params.put("relationTypeGroup", relationTypeGroup.name()); + + return given().spec(requestSpec) + .pathParams(params) + .get("/api/relations?fromId={fromId}&fromType={fromType}&relationTypeGroup={relationTypeGroup}") + .then() + .statusCode(HTTP_OK) + .extract() + .as(new TypeRef>() {}); + } + + public JsonNode postServerSideRpc(DeviceId deviceId, JsonNode serverRpcPayload) { + return given().spec(requestSpec) + .body(serverRpcPayload) + .post("/api/rpc/twoway/{deviceId}", deviceId.getId()) + .then() + .statusCode(HTTP_OK) + .extract() + .as(JsonNode.class); + } + + public String getToken() { + return token; + } + + public String getRefreshToken() { + return refreshToken; + } +} diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java index 979fbb187d..ed606cd468 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/ThingsBoardDbInstaller.java @@ -16,7 +16,6 @@ package org.thingsboard.server.msa; import lombok.extern.slf4j.Slf4j; -import org.junit.rules.ExternalResource; import org.testcontainers.utility.Base58; import org.thingsboard.server.common.data.StringUtils; @@ -30,7 +29,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; @Slf4j -public class ThingsBoardDbInstaller extends ExternalResource { +public class ThingsBoardDbInstaller { final static boolean IS_REDIS_CLUSTER = Boolean.parseBoolean(System.getProperty("blackBoxTests.redisCluster")); final static boolean IS_HYBRID_MODE = Boolean.parseBoolean(System.getProperty("blackBoxTests.hybridMode")); @@ -129,8 +128,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { return env; } - @Override - protected void before() throws Throwable { + public void createVolumes() { try { dockerCompose.withCommand("volume create " + postgresDataVolume); @@ -192,8 +190,7 @@ public class ThingsBoardDbInstaller extends ExternalResource { } } - @Override - protected void after() { + public void savaLogsAndRemoveVolumes() { copyLogs(tbLogVolume, "./target/tb-logs/"); copyLogs(tbCoapTransportLogVolume, "./target/tb-coap-transport-logs/"); copyLogs(tbLwm2mTransportLogVolume, "./target/tb-lwm2m-transport-logs/"); diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java index f81d03b394..2eb4cdf076 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/HttpClientTest.java @@ -16,109 +16,79 @@ package org.thingsboard.server.msa.connectivity; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Sets; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.http.ResponseEntity; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.msa.AbstractContainerTest; import org.thingsboard.server.msa.WsClient; import org.thingsboard.server.msa.mapper.WsTelemetryResponse; - -import java.util.Optional; +import java.util.Arrays; import java.util.concurrent.TimeUnit; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.assertj.core.api.Assertions.assertThat; import static org.thingsboard.server.common.data.DataConstants.DEVICE; import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; +import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; public class HttpClientTest extends AbstractContainerTest { + private Device device; + @BeforeMethod + public void setUp() throws Exception { + testRestClient.login("tenant@thingsboard.org", "tenant"); + device = testRestClient.postDevice("", defaultDevicePrototype("http_")); + } + + @AfterMethod + public void tearDown() { + testRestClient.deleteDeviceIfExists(device.getId()); + } @Test public void telemetryUpload() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - - Device device = createDevice("http_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); - ResponseEntity deviceTelemetryResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/v1/{credentialsId}/telemetry", - mapper.readTree(createPayload().toString()), - ResponseEntity.class, - deviceCredentials.getCredentialsId()); - Assert.assertTrue(deviceTelemetryResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetry(deviceCredentials.getCredentialsId(), mapper.readTree(createPayload().toString())); + WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); wsClient.closeBlocking(); - Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73))); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - restClient.deleteDevice(device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void getAttributes() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - TB_TOKEN = restClient.getToken(); + String accessToken = testRestClient.getDeviceCredentialsByDeviceId(device.getId()).getCredentialsId(); + assertThat(accessToken).isNotNull(); - Device device = createDevice("test"); - String accessToken = restClient.getDeviceCredentialsByDeviceId(device.getId()).get().getCredentialsId(); - assertNotNull(accessToken); + JsonNode sharedAattribute = mapper.readTree(createPayload().toString()); + testRestClient.postTelemetryAttribute(DEVICE, device.getId(), SHARED_SCOPE, sharedAattribute); - ResponseEntity deviceSharedAttributes = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/" + DEVICE + "/" + device.getId().toString() + "/attributes/" + SHARED_SCOPE, mapper.readTree(createPayload().toString()), - ResponseEntity.class, - accessToken); - - Assert.assertTrue(deviceSharedAttributes.getStatusCode().is2xxSuccessful()); - - ResponseEntity deviceClientsAttributes = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/v1/" + accessToken + "/attributes/", mapper.readTree(createPayload().toString()), - ResponseEntity.class, - accessToken); - - Assert.assertTrue(deviceClientsAttributes.getStatusCode().is2xxSuccessful()); + JsonNode clientAttribute = mapper.readTree(createPayload().toString()); + testRestClient.postAttribute(accessToken, clientAttribute); TimeUnit.SECONDS.sleep(3 * timeoutMultiplier); - @SuppressWarnings("deprecation") - Optional allOptional = restClient.getAttributes(accessToken, null, null); - assertTrue(allOptional.isPresent()); - - - JsonNode all = allOptional.get(); - assertEquals(2, all.size()); - assertEquals(mapper.readTree(createPayload().toString()), all.get("shared")); - assertEquals(mapper.readTree(createPayload().toString()), all.get("client")); - - @SuppressWarnings("deprecation") - Optional sharedOptional = restClient.getAttributes(accessToken, null, "stringKey"); - assertTrue(sharedOptional.isPresent()); - - JsonNode shared = sharedOptional.get(); - assertEquals(shared.get("shared").get("stringKey"), mapper.readTree(createPayload().get("stringKey").toString())); - assertFalse(shared.has("client")); + JsonNode attributes = testRestClient.getAttributes(accessToken, null, null); + assertThat(attributes.get("shared")).isEqualTo(sharedAattribute); + assertThat(attributes.get("client")).isEqualTo(clientAttribute); - @SuppressWarnings("deprecation") - Optional clientOptional = restClient.getAttributes(accessToken, "longKey,stringKey", null); - assertTrue(clientOptional.isPresent()); + JsonNode attributes2 = testRestClient.getAttributes(accessToken, null, "stringKey"); + assertThat(attributes2.get("shared").get("stringKey")).isEqualTo(sharedAattribute.get("stringKey")); + assertThat(attributes2.has("client")).isFalse(); - JsonNode client = clientOptional.get(); - assertFalse(client.has("shared")); - assertEquals(mapper.readTree(createPayload().get("longKey").toString()), client.get("client").get("longKey")); - assertEquals(client.get("client").get("stringKey"), mapper.readTree(createPayload().get("stringKey").toString())); + JsonNode attributes3 = testRestClient.getAttributes(accessToken, "longKey,stringKey", null); - restClient.deleteDevice(device.getId()); + assertThat(attributes3.has("shared")).isFalse(); + assertThat(attributes3.get("client").get("longKey")).isEqualTo(clientAttribute.get("longKey")); + assertThat(attributes3.get("client").get("stringKey")).isEqualTo(clientAttribute.get("stringKey")); } } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java index ce36f08a64..55dd432644 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttClientTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.msa.connectivity; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -26,19 +25,19 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.junit.Test; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpMethod; -import org.springframework.http.ResponseEntity; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.RuleChainId; import org.thingsboard.server.common.data.page.PageData; +import org.thingsboard.server.common.data.page.PageLink; import org.thingsboard.server.common.data.rule.NodeConnectionInfo; import org.thingsboard.server.common.data.rule.RuleChain; import org.thingsboard.server.common.data.rule.RuleChainMetaData; @@ -61,14 +60,29 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; +import static org.testng.Assert.fail; +import static org.thingsboard.server.common.data.DataConstants.DEVICE; +import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; +import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultDevicePrototype; + @Slf4j public class MqttClientTest extends AbstractContainerTest { + private Device device; + @BeforeMethod + public void setUp() throws Exception { + testRestClient.login("tenant@thingsboard.org", "tenant"); + device = testRestClient.postDevice("", defaultDevicePrototype("http_")); + } + + @AfterMethod + public void tearDown() { + testRestClient.deleteDeviceIfExists(device.getId()); + } @Test public void telemetryUpload() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); MqttClient mqttClient = getMqttClient(deviceCredentials, null); @@ -77,25 +91,19 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"), - actualLatestTelemetry.getLatestValues().keySet()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73))); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void telemetryUploadWithTs() throws Exception { long ts = 1451649600512L; - - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); MqttClient mqttClient = getMqttClient(deviceCredentials, null); @@ -104,22 +112,18 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(getExpectedLatestValues(ts), actualLatestTelemetry.getLatestValues()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(getExpectedLatestValues(ts)).isEqualTo(actualLatestTelemetry.getLatestValues()); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", ts, Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", ts, "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", ts, Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", ts, Long.toString(73))); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void publishAttributeUpdateToServer() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); MqttMessageListener listener = new MqttMessageListener(); @@ -134,23 +138,18 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("attr1", "attr2", "attr3", "attr4"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "attr1", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "attr2", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "attr3", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "attr4", Long.toString(73))); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("attr1", "attr2", "attr3", "attr4")); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr1").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr2").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr3").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr4").get(1)).isEqualTo(Long.toString(73)); } @Test public void requestAttributeValuesFromServer() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); WsClient wsClient = subscribeToWebSocket(device.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); MqttMessageListener listener = new MqttMessageListener(); @@ -166,21 +165,16 @@ public class MqttClientTest extends AbstractContainerTest { log.info("Received ws telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(1, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("clientAttr"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "clientAttr", clientAttributeValue)); + assertThat(actualLatestTelemetry.getData()).hasSize(1); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnly("clientAttr"); + assertThat(actualLatestTelemetry.getDataValuesByKey("clientAttr").get(1)).isEqualTo(clientAttributeValue); // Add a new shared attribute JsonObject sharedAttributes = new JsonObject(); String sharedAttributeValue = StringUtils.randomAlphanumeric(8); sharedAttributes.addProperty("sharedAttr", sharedAttributeValue); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - device.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + JsonNode sharedAttribute = mapper.readTree(sharedAttributes.toString()); + testRestClient.postTelemetryAttribute(DataConstants.DEVICE, device.getId(), SHARED_SCOPE, sharedAttribute); // Subscribe to attributes response mqttClient.on("v1/devices/me/attributes/response/+", listener, MqttQoS.AT_LEAST_ONCE).get(); @@ -197,20 +191,16 @@ public class MqttClientTest extends AbstractContainerTest { AttributesResponse attributes = mapper.readValue(Objects.requireNonNull(event).getMessage(), AttributesResponse.class); log.info("Received telemetry: {}", attributes); - Assert.assertEquals(1, attributes.getClient().size()); - Assert.assertEquals(clientAttributeValue, attributes.getClient().get("clientAttr")); - - Assert.assertEquals(1, attributes.getShared().size()); - Assert.assertEquals(sharedAttributeValue, attributes.getShared().get("sharedAttr")); + assertThat(attributes.getClient()).hasSize(1); + assertThat(attributes.getClient().get("clientAttr")).isEqualTo(clientAttributeValue); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(attributes.getShared()).hasSize(1); + assertThat(attributes.getShared().get("sharedAttr")).isEqualTo(sharedAttributeValue); } @Test public void subscribeToAttributeUpdatesFromServer() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); @@ -225,38 +215,28 @@ public class MqttClientTest extends AbstractContainerTest { JsonObject sharedAttributes = new JsonObject(); String sharedAttributeValue = StringUtils.randomAlphanumeric(8); sharedAttributes.addProperty(sharedAttributeName, sharedAttributeValue); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - device.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + JsonNode sharedAttribute = mapper.readTree(sharedAttributes.toString()); + + testRestClient.postTelemetryAttribute(DataConstants.DEVICE, device.getId(), SHARED_SCOPE, sharedAttribute); MqttEvent event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(sharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()) + .isEqualTo(sharedAttributeValue); // Update the shared attribute value JsonObject updatedSharedAttributes = new JsonObject(); String updatedSharedAttributeValue = StringUtils.randomAlphanumeric(8); updatedSharedAttributes.addProperty(sharedAttributeName, updatedSharedAttributeValue); - ResponseEntity updatedSharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(updatedSharedAttributes.toString()), ResponseEntity.class, - device.getId()); - Assert.assertTrue(updatedSharedAttributesResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetryAttribute(DEVICE, device.getId(), SHARED_SCOPE, mapper.readTree(updatedSharedAttributes.toString())); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(updatedSharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get(sharedAttributeName).asText()) + .isEqualTo(updatedSharedAttributeValue); } @Test public void serverSideRpc() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); @@ -270,21 +250,18 @@ public class MqttClientTest extends AbstractContainerTest { serverRpcPayload.addProperty("method", "getValue"); serverRpcPayload.addProperty("params", true); ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(getClass().getSimpleName()))); - ListenableFuture future = service.submit(() -> { + ListenableFuture future = service.submit(() -> { try { - return restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/rpc/twoway/{deviceId}", - mapper.readTree(serverRpcPayload.toString()), String.class, - device.getId()); + return testRestClient.postServerSideRpc(device.getId(), mapper.readTree(serverRpcPayload.toString())); } catch (IOException e) { - return ResponseEntity.badRequest().build(); + return null; } }); // Wait for RPC call from the server and send the response MqttEvent requestFromServer = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals("{\"method\":\"getValue\",\"params\":true}", Objects.requireNonNull(requestFromServer).getMessage()); + assertThat(Objects.requireNonNull(requestFromServer).getMessage()).isEqualTo("{\"method\":\"getValue\",\"params\":true}"); Integer requestId = Integer.valueOf(Objects.requireNonNull(requestFromServer).getTopic().substring("v1/devices/me/rpc/request/".length())); JsonObject clientResponse = new JsonObject(); @@ -292,19 +269,14 @@ public class MqttClientTest extends AbstractContainerTest { // Send a response to the server's RPC request mqttClient.publish("v1/devices/me/rpc/response/" + requestId, Unpooled.wrappedBuffer(clientResponse.toString().getBytes())).get(); - ResponseEntity serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); + JsonNode serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); service.shutdownNow(); - Assert.assertTrue(serverResponse.getStatusCode().is2xxSuccessful()); - Assert.assertEquals(clientResponse.toString(), serverResponse.getBody()); - - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + assertThat(serverResponse).isEqualTo(mapper.readTree(clientResponse.toString())); } @Test public void clientSideRpc() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - Device device = createDevice("mqtt_"); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); @@ -328,46 +300,33 @@ public class MqttClientTest extends AbstractContainerTest { TimeUnit.SECONDS.sleep(1 * timeoutMultiplier); MqttEvent responseFromServer = listener.getEvents().poll(1 * timeoutMultiplier, TimeUnit.SECONDS); Integer responseId = Integer.valueOf(Objects.requireNonNull(responseFromServer).getTopic().substring("v1/devices/me/rpc/response/".length())); - Assert.assertEquals(requestId, responseId); - Assert.assertEquals("requestReceived", mapper.readTree(responseFromServer.getMessage()).get("response").asText()); + assertThat(responseId).isEqualTo(requestId); + assertThat(mapper.readTree(responseFromServer.getMessage()).get("response").asText()).isEqualTo("requestReceived"); // Make the default rule chain a root again - ResponseEntity rootRuleChainResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain/{ruleChainId}/root", - null, - RuleChain.class, - defaultRuleChainId); - Assert.assertTrue(rootRuleChainResponse.getStatusCode().is2xxSuccessful()); + testRestClient.setRootRuleChain(defaultRuleChainId); // Delete the created rule chain - restClient.getRestTemplate().delete(HTTPS_URL + "/api/ruleChain/{ruleChainId}", ruleChainId); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + device.getId()); + testRestClient.deleteRuleChain(ruleChainId); } @Test public void deviceDeletedClosingSession() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - String deviceForDeletingTestName = "Device for deleting notification test"; - Device device = createDevice(deviceForDeletingTestName); - DeviceCredentials deviceCredentials = restClient.getDeviceCredentialsByDeviceId(device.getId()).get(); + DeviceCredentials deviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(device.getId()); MqttMessageListener listener = new MqttMessageListener(); MqttClient mqttClient = getMqttClient(deviceCredentials, listener); - restClient.deleteDevice(device.getId()); + testRestClient.deleteDeviceIfExists(device.getId()); TimeUnit.SECONDS.sleep(3 * timeoutMultiplier); - Assert.assertFalse(mqttClient.isConnected()); + assertThat(mqttClient.isConnected()).isFalse(); } private RuleChainId createRootRuleChainForRpcResponse() throws Exception { RuleChain newRuleChain = new RuleChain(); newRuleChain.setName("testRuleChain"); - ResponseEntity ruleChainResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain", - newRuleChain, - RuleChain.class); - Assert.assertTrue(ruleChainResponse.getStatusCode().is2xxSuccessful()); - RuleChain ruleChain = ruleChainResponse.getBody(); + + RuleChain ruleChain = testRestClient.postRootRuleChain(newRuleChain); JsonNode configuration = mapper.readTree(this.getClass().getClassLoader().getResourceAsStream("RpcResponseRuleChainMetadata.json")); RuleChainMetaData ruleChainMetaData = new RuleChainMetaData(); @@ -376,37 +335,22 @@ public class MqttClientTest extends AbstractContainerTest { ruleChainMetaData.setNodes(Arrays.asList(mapper.treeToValue(configuration.get("nodes"), RuleNode[].class))); ruleChainMetaData.setConnections(Arrays.asList(mapper.treeToValue(configuration.get("connections"), NodeConnectionInfo[].class))); - ResponseEntity ruleChainMetadataResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain/metadata", - ruleChainMetaData, - RuleChainMetaData.class); - Assert.assertTrue(ruleChainMetadataResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postRuleChainMetadata(ruleChainMetaData); // Set a new rule chain as root - ResponseEntity rootRuleChainResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/ruleChain/{ruleChainId}/root", - null, - RuleChain.class, - ruleChain.getId()); - Assert.assertTrue(rootRuleChainResponse.getStatusCode().is2xxSuccessful()); - + testRestClient.setRootRuleChain(ruleChain.getId()); return ruleChain.getId(); } private RuleChainId getDefaultRuleChainId() { - ResponseEntity> ruleChains = restClient.getRestTemplate().exchange( - HTTPS_URL + "/api/ruleChains?pageSize=40&page=0&textSearch=", - HttpMethod.GET, - null, - new ParameterizedTypeReference>() { - }); - - Optional defaultRuleChain = ruleChains.getBody().getData() + PageData ruleChains = testRestClient.getRuleChains(new PageLink(40, 0)); + + Optional defaultRuleChain = ruleChains.getData() .stream() .filter(RuleChain::isRoot) .findFirst(); if (!defaultRuleChain.isPresent()) { - Assert.fail("Root rule chain wasn't found"); + fail("Root rule chain wasn't found"); } return defaultRuleChain.get().getId(); } diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java index 25a8946431..84b2dd7253 100644 --- a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/connectivity/MqttGatewayClientTest.java @@ -16,7 +16,6 @@ package org.thingsboard.server.msa.connectivity; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.Sets; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; @@ -28,15 +27,17 @@ import io.netty.buffer.Unpooled; import io.netty.handler.codec.mqtt.MqttQoS; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.springframework.http.ResponseEntity; +import org.springframework.http.HttpStatus; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.common.util.ThingsBoardThreadFactory; import org.thingsboard.mqtt.MqttClient; import org.thingsboard.mqtt.MqttClientConfig; import org.thingsboard.mqtt.MqttHandler; +import org.thingsboard.server.common.data.DataConstants; import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.StringUtils; import org.thingsboard.server.common.data.id.DeviceId; @@ -50,9 +51,9 @@ import org.thingsboard.server.msa.mapper.WsTelemetryResponse; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.Random; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -60,28 +61,34 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; +import static org.thingsboard.server.common.data.DataConstants.DEVICE; +import static org.thingsboard.server.common.data.DataConstants.SHARED_SCOPE; +import static org.thingsboard.server.msa.prototypes.DevicePrototypes.defaultGatewayPrototype; + @Slf4j public class MqttGatewayClientTest extends AbstractContainerTest { - Device gatewayDevice; - MqttClient mqttClient; - Device createdDevice; - MqttMessageListener listener; + private Device gatewayDevice; + private MqttClient mqttClient; + private Device createdDevice; + private MqttMessageListener listener; + private JsonParser jsonParser = new JsonParser(); - @Before + @BeforeMethod public void createGateway() throws Exception { - restClient.login("tenant@thingsboard.org", "tenant"); - this.gatewayDevice = createGatewayDevice(); - Optional gatewayDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(gatewayDevice.getId()); - Assert.assertTrue(gatewayDeviceCredentials.isPresent()); + testRestClient.login("tenant@thingsboard.org", "tenant"); + gatewayDevice = testRestClient.postDevice("", defaultGatewayPrototype()); + DeviceCredentials gatewayDeviceCredentials = testRestClient.getDeviceCredentialsByDeviceId(gatewayDevice.getId()); + this.listener = new MqttMessageListener(); - this.mqttClient = getMqttClient(gatewayDeviceCredentials.get(), listener); + this.mqttClient = getMqttClient(gatewayDeviceCredentials, listener); this.createdDevice = createDeviceThroughGateway(mqttClient, gatewayDevice); } - @After - public void removeGateway() throws Exception { - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.gatewayDevice.getId()); - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.createdDevice.getId()); + @AfterMethod + public void removeGateway() { + testRestClient.deleteDeviceIfExists(this.gatewayDevice.getId()); + testRestClient.deleteDeviceIfExists(this.createdDevice.getId()); this.listener = null; this.mqttClient = null; this.createdDevice = null; @@ -95,40 +102,38 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("booleanKey", "stringKey", "doubleKey", "longKey"), - actualLatestTelemetry.getLatestValues().keySet()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", Long.toString(73))); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void telemetryUploadWithTs() throws Exception { long ts = 1451649600512L; - restClient.login("tenant@thingsboard.org", "tenant"); WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "LATEST_TELEMETRY", CmdsType.TS_SUB_CMDS); mqttClient.publish("v1/gateway/telemetry", Unpooled.wrappedBuffer(createGatewayPayload(createdDevice.getName(), ts).toString().getBytes())).get(); WsTelemetryResponse actualLatestTelemetry = wsClient.getLastMessage(); log.info("Received telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(getExpectedLatestValues(ts), actualLatestTelemetry.getLatestValues()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("booleanKey", "stringKey", "doubleKey", "longKey")); - Assert.assertTrue(verify(actualLatestTelemetry, "booleanKey", ts, Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "stringKey", ts, "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "doubleKey", ts, Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "longKey", ts, Long.toString(73))); + assertThat(actualLatestTelemetry.getDataValuesByKey("booleanKey").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("stringKey").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("doubleKey").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("longKey").get(1)).isEqualTo(Long.toString(73)); } @Test public void publishAttributeUpdateToServer() throws Exception { - Optional createdDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); - Assert.assertTrue(createdDeviceCredentials.isPresent()); + testRestClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); + WsClient wsClient = subscribeToWebSocket(createdDevice.getId(), "CLIENT_SCOPE", CmdsType.ATTR_SUB_CMDS); JsonObject clientAttributes = new JsonObject(); clientAttributes.addProperty("attr1", "value1"); @@ -142,20 +147,18 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info("Received attributes: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(4, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("attr1", "attr2", "attr3", "attr4"), - actualLatestTelemetry.getLatestValues().keySet()); + assertThat(actualLatestTelemetry.getData()).hasSize(4); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnlyOnceElementsOf(Arrays.asList("attr1", "attr2", "attr3", "attr4")); - Assert.assertTrue(verify(actualLatestTelemetry, "attr1", "value1")); - Assert.assertTrue(verify(actualLatestTelemetry, "attr2", Boolean.TRUE.toString())); - Assert.assertTrue(verify(actualLatestTelemetry, "attr3", Double.toString(42.0))); - Assert.assertTrue(verify(actualLatestTelemetry, "attr4", Long.toString(73))); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr1").get(1)).isEqualTo("value1"); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr2").get(1)).isEqualTo(Boolean.TRUE.toString()); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr3").get(1)).isEqualTo(Double.toString(42.0)); + assertThat(actualLatestTelemetry.getDataValuesByKey("attr4").get(1)).isEqualTo(Long.toString(73)); } @Test public void responseDataOnAttributesRequestCheck() throws Exception { - Optional createdDeviceCredentials = restClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); - Assert.assertTrue(createdDeviceCredentials.isPresent()); + testRestClient.getDeviceCredentialsByDeviceId(createdDevice.getId()); JsonObject sharedAttributes = new JsonObject(); sharedAttributes.addProperty("attr1", "value1"); sharedAttributes.addProperty("attr2", true); @@ -163,11 +166,8 @@ public class MqttGatewayClientTest extends AbstractContainerTest { sharedAttributes.addProperty("attr4", 73); mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + + testRestClient.postTelemetryAttribute(DataConstants.DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(sharedAttributes.toString())); var event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); JsonObject requestData = new JsonObject(); @@ -181,8 +181,8 @@ public class MqttGatewayClientTest extends AbstractContainerTest { event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); JsonObject responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); - Assert.assertTrue(responseData.has("value")); - Assert.assertEquals(sharedAttributes.get("attr1").getAsString(), responseData.get("value").getAsString()); + assertThat(responseData.has("value")).isTrue(); + assertThat(responseData.get("value").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); requestData = new JsonObject(); requestData.addProperty("id", 1); @@ -198,9 +198,9 @@ public class MqttGatewayClientTest extends AbstractContainerTest { event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); - Assert.assertTrue(responseData.has("values")); - Assert.assertEquals(sharedAttributes.get("attr1").getAsString(), responseData.get("values").getAsJsonObject().get("attr1").getAsString()); - Assert.assertEquals(sharedAttributes.get("attr2").getAsString(), responseData.get("values").getAsJsonObject().get("attr2").getAsString()); + assertThat(responseData.has("values")).isTrue(); + assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); + assertThat(responseData.get("values").getAsJsonObject().get("attr2").getAsString()).isEqualTo(sharedAttributes.get("attr2").getAsString()); requestData = new JsonObject(); requestData.addProperty("id", 1); @@ -216,9 +216,9 @@ public class MqttGatewayClientTest extends AbstractContainerTest { event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); responseData = jsonParser.parse(Objects.requireNonNull(event).getMessage()).getAsJsonObject(); - Assert.assertTrue(responseData.has("values")); - Assert.assertEquals(sharedAttributes.get("attr1").getAsString(), responseData.get("values").getAsJsonObject().get("attr1").getAsString()); - Assert.assertEquals(1, responseData.get("values").getAsJsonObject().entrySet().size()); + assertThat(responseData.has("values")).isTrue(); + assertThat(responseData.get("values").getAsJsonObject().get("attr1").getAsString()).isEqualTo(sharedAttributes.get("attr1").getAsString()); + assertThat(responseData.get("values").getAsJsonObject().entrySet()).hasSize(1); } @Test @@ -237,11 +237,9 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info("Received ws telemetry: {}", actualLatestTelemetry); wsClient.closeBlocking(); - Assert.assertEquals(1, actualLatestTelemetry.getData().size()); - Assert.assertEquals(Sets.newHashSet("clientAttr"), - actualLatestTelemetry.getLatestValues().keySet()); - - Assert.assertTrue(verify(actualLatestTelemetry, "clientAttr", clientAttributeValue)); + assertThat(actualLatestTelemetry.getData()).hasSize(1); + assertThat(actualLatestTelemetry.getLatestValues().keySet()).containsOnly("clientAttr"); + assertThat(actualLatestTelemetry.getDataValuesByKey("clientAttr").get(1)).isEqualTo(clientAttributeValue); // Add a new shared attribute JsonObject sharedAttributes = new JsonObject(); @@ -251,16 +249,12 @@ public class MqttGatewayClientTest extends AbstractContainerTest { // Subscribe for attribute update event mqttClient.on("v1/gateway/attributes", listener, MqttQoS.AT_LEAST_ONCE).get(); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetryAttribute(DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(sharedAttributes.toString())); MqttEvent sharedAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); // Catch attribute update event - Assert.assertNotNull(sharedAttributeEvent); - Assert.assertEquals("v1/gateway/attributes", sharedAttributeEvent.getTopic()); + assertThat(sharedAttributeEvent).isNotNull(); + assertThat(sharedAttributeEvent.getTopic()).isEqualTo("v1/gateway/attributes"); // Subscribe to attributes response mqttClient.on("v1/gateway/attributes/response", listener, MqttQoS.AT_LEAST_ONCE).get(); @@ -288,15 +282,11 @@ public class MqttGatewayClientTest extends AbstractContainerTest { gatewaySharedAttributeValue.addProperty("device", createdDevice.getName()); gatewaySharedAttributeValue.add("data", sharedAttributes); - ResponseEntity sharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(sharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(sharedAttributesResponse.getStatusCode().is2xxSuccessful()); + testRestClient.postTelemetryAttribute(DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(sharedAttributes.toString())); MqttEvent event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(sharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()) + .isEqualTo(sharedAttributeValue); // Update the shared attribute value JsonObject updatedSharedAttributes = new JsonObject(); @@ -307,15 +297,10 @@ public class MqttGatewayClientTest extends AbstractContainerTest { gatewayUpdatedSharedAttributeValue.addProperty("device", createdDevice.getName()); gatewayUpdatedSharedAttributeValue.add("data", updatedSharedAttributes); - ResponseEntity updatedSharedAttributesResponse = restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/plugins/telemetry/DEVICE/{deviceId}/SHARED_SCOPE", - mapper.readTree(updatedSharedAttributes.toString()), ResponseEntity.class, - createdDevice.getId()); - Assert.assertTrue(updatedSharedAttributesResponse.getStatusCode().is2xxSuccessful()); - + testRestClient.postTelemetryAttribute(DEVICE, createdDevice.getId(), SHARED_SCOPE, mapper.readTree(updatedSharedAttributes.toString())); event = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertEquals(updatedSharedAttributeValue, - mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()); + assertThat(mapper.readValue(Objects.requireNonNull(event).getMessage(), JsonNode.class).get("data").get(sharedAttributeName).asText()) + .isEqualTo(updatedSharedAttributeValue); } @Test @@ -331,14 +316,11 @@ public class MqttGatewayClientTest extends AbstractContainerTest { serverRpcPayload.addProperty("method", "getValue"); serverRpcPayload.addProperty("params", true); ListeningExecutorService service = MoreExecutors.listeningDecorator(Executors.newSingleThreadExecutor(ThingsBoardThreadFactory.forName(getClass().getSimpleName()))); - ListenableFuture future = service.submit(() -> { + ListenableFuture future = service.submit(() -> { try { - return restClient.getRestTemplate() - .postForEntity(HTTPS_URL + "/api/rpc/twoway/{deviceId}", - mapper.readTree(serverRpcPayload.toString()), String.class, - createdDevice.getId()); + return testRestClient.postServerSideRpc(createdDevice.getId(), mapper.readTree(serverRpcPayload.toString())); } catch (IOException e) { - return ResponseEntity.badRequest().build(); + return null; } }); @@ -346,19 +328,13 @@ public class MqttGatewayClientTest extends AbstractContainerTest { MqttEvent requestFromServer = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); service.shutdownNow(); - Assert.assertNotNull(requestFromServer); - Assert.assertNotNull(requestFromServer.getMessage()); - - JsonObject requestFromServerJson = new JsonParser().parse(requestFromServer.getMessage()).getAsJsonObject(); - - Assert.assertEquals(createdDevice.getName(), requestFromServerJson.get("device").getAsString()); - - JsonObject requestFromServerData = requestFromServerJson.get("data").getAsJsonObject(); - - Assert.assertEquals("getValue", requestFromServerData.get("method").getAsString()); - Assert.assertTrue(requestFromServerData.get("params").getAsBoolean()); - - int requestId = requestFromServerData.get("id").getAsInt(); + assertThat(requestFromServer).isNotNull(); + assertThat(requestFromServer.getMessage()).isNotNull(); + JsonNode requestFromServerJson = JacksonUtil.toJsonNode(requestFromServer.getMessage()); + assertThat(requestFromServerJson.get("device").asText()).isEqualTo(createdDevice.getName()); + assertThat(requestFromServerJson.get("data").get("method").asText()).isEqualTo("getValue"); + assertThat(requestFromServerJson.get("data").get("params").asText()).isEqualTo("true"); + int requestId = requestFromServerJson.get("data").get("id").asInt(); JsonObject clientResponse = new JsonObject(); clientResponse.addProperty("response", "someResponse"); @@ -369,16 +345,15 @@ public class MqttGatewayClientTest extends AbstractContainerTest { // Send a response to the server's RPC request mqttClient.publish(gatewayRpcTopic, Unpooled.wrappedBuffer(gatewayResponse.toString().getBytes())).get(); - ResponseEntity serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertTrue(serverResponse.getStatusCode().is2xxSuccessful()); - Assert.assertEquals(clientResponse.toString(), serverResponse.getBody()); + JsonNode serverResponse = future.get(5 * timeoutMultiplier, TimeUnit.SECONDS); + + assertThat(serverResponse).isEqualTo(mapper.readTree(clientResponse.toString())); } @Test public void deviceCreationAfterDeleted() throws Exception { - restClient.getRestTemplate().delete(HTTPS_URL + "/api/device/" + this.createdDevice.getId()); - Optional deletedDevice = restClient.getDeviceById(this.createdDevice.getId()); - Assert.assertTrue(deletedDevice.isEmpty()); + testRestClient.deleteDevice(this.createdDevice.getId()); + testRestClient.getDeviceById(this.createdDevice.getId(), HttpStatus.NOT_FOUND.value()); this.createdDevice = createDeviceThroughGateway(mqttClient, gatewayDevice); } @@ -397,13 +372,13 @@ public class MqttGatewayClientTest extends AbstractContainerTest { log.info(gatewayAttributesRequest.toString()); mqttClient.publish("v1/gateway/attributes/request", Unpooled.wrappedBuffer(gatewayAttributesRequest.toString().getBytes())).get(); MqttEvent clientAttributeEvent = listener.getEvents().poll(10 * timeoutMultiplier, TimeUnit.SECONDS); - Assert.assertNotNull(clientAttributeEvent); + assertThat(clientAttributeEvent).isNotNull(); JsonObject responseMessage = new JsonParser().parse(Objects.requireNonNull(clientAttributeEvent).getMessage()).getAsJsonObject(); - Assert.assertEquals(messageId, responseMessage.get("id").getAsInt()); - Assert.assertEquals(createdDevice.getName(), responseMessage.get("device").getAsString()); - Assert.assertEquals(3, responseMessage.entrySet().size()); - Assert.assertEquals(expectedValue, responseMessage.get("value").getAsString()); + assertThat(responseMessage.get("id").getAsInt()).isEqualTo(messageId); + assertThat(responseMessage.get("device").getAsString()).isEqualTo(createdDevice.getName()); + assertThat(responseMessage.entrySet()).hasSize(3); + assertThat(responseMessage.get("value").getAsString()).isEqualTo(expectedValue); } private Device createDeviceThroughGateway(MqttClient mqttClient, Device gatewayDevice) throws Exception { @@ -411,24 +386,19 @@ public class MqttGatewayClientTest extends AbstractContainerTest { TimeUnit.SECONDS.sleep(30); } - String deviceName = "mqtt_device"; + String deviceName = "mqtt_device" + RandomStringUtils.randomAlphabetic(5); mqttClient.publish("v1/gateway/connect", Unpooled.wrappedBuffer(createGatewayConnectPayload(deviceName).toString().getBytes()), MqttQoS.AT_LEAST_ONCE).get(); if (timeoutMultiplier > 1) { TimeUnit.SECONDS.sleep(30); } - List relations = restClient.findByFrom(gatewayDevice.getId(), RelationTypeGroup.COMMON); - - Assert.assertEquals(1, relations.size()); + List relations = testRestClient.findRelationByFrom(gatewayDevice.getId(), RelationTypeGroup.COMMON); + assertThat(relations).hasSize(1); EntityId createdEntityId = relations.get(0).getTo(); DeviceId createdDeviceId = new DeviceId(createdEntityId.getId()); - Optional createdDevice = restClient.getDeviceById(createdDeviceId); - - Assert.assertTrue(createdDevice.isPresent()); - - return createdDevice.get(); + return testRestClient.getDeviceById(createdDeviceId); } private MqttClient getMqttClient(DeviceCredentials deviceCredentials, MqttMessageListener listener) throws InterruptedException, ExecutionException { diff --git a/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java new file mode 100644 index 0000000000..7db35460a9 --- /dev/null +++ b/msa/black-box-tests/src/test/java/org/thingsboard/server/msa/prototypes/DevicePrototypes.java @@ -0,0 +1,40 @@ +/** + * Copyright © 2016-2022 The Thingsboard Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.thingsboard.server.msa.prototypes; + +import com.fasterxml.jackson.databind.JsonNode; +import org.testcontainers.shaded.org.apache.commons.lang3.RandomStringUtils; +import org.thingsboard.common.util.JacksonUtil; +import org.thingsboard.server.common.data.Device; + +public class DevicePrototypes { + public static Device defaultDevicePrototype(String name){ + Device device = new Device(); + device.setName(name + RandomStringUtils.randomAlphanumeric(7)); + device.setType("DEFAULT"); + return device; + } + + public static Device defaultGatewayPrototype() { + String isGateway = "{\"gateway\":true}"; + JsonNode additionalInfo = JacksonUtil.toJsonNode(isGateway); + Device gatewayDeviceTemplate = new Device(); + gatewayDeviceTemplate.setName("mqtt_gateway_" + RandomStringUtils.randomAlphanumeric(5)); + gatewayDeviceTemplate.setType("gateway"); + gatewayDeviceTemplate.setAdditionalInfo(additionalInfo); + return gatewayDeviceTemplate; + } +} diff --git a/msa/black-box-tests/src/test/resources/config.properties b/msa/black-box-tests/src/test/resources/config.properties new file mode 100644 index 0000000000..419c73185d --- /dev/null +++ b/msa/black-box-tests/src/test/resources/config.properties @@ -0,0 +1,2 @@ +tb.baseUrl=http://localhost:8080 +tb.wsUrl=ws://localhost:8080 diff --git a/msa/black-box-tests/src/test/resources/testNG.xml b/msa/black-box-tests/src/test/resources/testNG.xml new file mode 100644 index 0000000000..45e93f76f1 --- /dev/null +++ b/msa/black-box-tests/src/test/resources/testNG.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/msa/js-executor/api/jsExecutor.models.ts b/msa/js-executor/api/jsExecutor.models.ts index 7a6b53cd8a..17407f4d50 100644 --- a/msa/js-executor/api/jsExecutor.models.ts +++ b/msa/js-executor/api/jsExecutor.models.ts @@ -56,7 +56,7 @@ export interface JsCompileResponse extends TbMessage { export interface JsInvokeResponse { success: boolean; - result: string; + result?: string; errorCode?: number; errorDetails?: string; } diff --git a/msa/js-executor/api/jsInvokeMessageProcessor.ts b/msa/js-executor/api/jsInvokeMessageProcessor.ts index 668cd61f50..52a337c74a 100644 --- a/msa/js-executor/api/jsInvokeMessageProcessor.ts +++ b/msa/js-executor/api/jsInvokeMessageProcessor.ts @@ -39,6 +39,7 @@ const TIMEOUT_ERROR = 2; const NOT_FOUND_ERROR = 3; const statFrequency = Number(config.get('script.stat_print_frequency')); +const memoryUsageTraceFrequency = Number(config.get('script.memory_usage_trace_frequency')); const scriptBodyTraceFrequency = Number(config.get('script.script_body_trace_frequency')); const useSandbox = config.get('script.use_sandbox') === 'true'; const maxActiveScripts = Number(config.get('script.max_active_scripts')); @@ -167,11 +168,15 @@ export class JsInvokeMessageProcessor { if (this.executedScriptsCounter % scriptBodyTraceFrequency == 0) { this.logger.info('[%s] Executing script body: [%s]', scriptId, invokeRequest.scriptBody); } + if (this.executedScriptsCounter % memoryUsageTraceFrequency == 0) { + this.logger.info('Current memory usage: [%s]', process.memoryUsage()); + } + this.getOrCompileScript(scriptId, invokeRequest.scriptBody).then( (script) => { this.executor.executeScript(script, invokeRequest.args, invokeRequest.timeout).then( - (result) => { - if (result.length <= maxResultSize) { + (result: string | undefined) => { + if (!result || result.length <= maxResultSize) { const invokeResponse = JsInvokeMessageProcessor.createInvokeResponse(result, true); this.logger.debug('[%s] Sending success invoke response, scriptId: [%s]', requestId, scriptId); this.sendResponse(requestId, responseTopic, headers, scriptId, undefined, invokeResponse); @@ -323,7 +328,7 @@ export class JsInvokeMessageProcessor { } } - private static createInvokeResponse(result: string, success: boolean, errorCode?: number, err?: any): JsInvokeResponse { + private static createInvokeResponse(result: string | undefined, success: boolean, errorCode?: number, err?: any): JsInvokeResponse { return { errorCode: errorCode, success: success, diff --git a/msa/js-executor/config/custom-environment-variables.yml b/msa/js-executor/config/custom-environment-variables.yml index b9c24c8d8d..2ebea4ccc1 100644 --- a/msa/js-executor/config/custom-environment-variables.yml +++ b/msa/js-executor/config/custom-environment-variables.yml @@ -75,6 +75,7 @@ logger: script: use_sandbox: "SCRIPT_USE_SANDBOX" + memory_usage_trace_frequency: "MEMORY_USAGE_TRACE_FREQUENCY" stat_print_frequency: "SCRIPT_STAT_PRINT_FREQUENCY" script_body_trace_frequency: "SCRIPT_BODY_TRACE_FREQUENCY" max_active_scripts: "MAX_ACTIVE_SCRIPTS" diff --git a/msa/js-executor/config/default.yml b/msa/js-executor/config/default.yml index 64829ef792..96f3401da5 100644 --- a/msa/js-executor/config/default.yml +++ b/msa/js-executor/config/default.yml @@ -64,6 +64,7 @@ logger: script: use_sandbox: "true" + memory_usage_trace_frequency: "1000" script_body_trace_frequency: "10000" stat_print_frequency: "10000" max_active_scripts: "1000" diff --git a/msa/js-executor/docker/start-js-executor.sh b/msa/js-executor/docker/start-js-executor.sh index 575f93c389..d30b62c145 100755 --- a/msa/js-executor/docker/start-js-executor.sh +++ b/msa/js-executor/docker/start-js-executor.sh @@ -27,4 +27,4 @@ source "${CONF_FOLDER}/${configfile}" cd ${pkg.installFolder} # This will forward this PID 1 to the node.js and forward SIGTERM for graceful shutdown as well -exec node server.js +exec node --no-compilation-cache server.js diff --git a/packaging/java/scripts/install/logback.xml b/packaging/java/scripts/install/logback.xml index 0047956c93..9233ab4d0b 100644 --- a/packaging/java/scripts/install/logback.xml +++ b/packaging/java/scripts/install/logback.xml @@ -56,6 +56,10 @@ + + + + diff --git a/pom.xml b/pom.xml index 60e8783bca..411e9be7cc 100755 --- a/pom.xml +++ b/pom.xml @@ -77,7 +77,7 @@ 3.5.5 3.21.9 1.42.1 - 2.4.23TB + 2.4.25TB 1.18.18 1.2.4 4.1.75.Final @@ -134,6 +134,10 @@ 1.3.0 1.2.7 + 7.6.1 + 3.23.1 + 5.2.0 + 1.3 1.17.3 1.12 3.0.0 @@ -1634,6 +1638,30 @@ + + org.testng + testng + ${testng.version} + test + + + org.assertj + assertj-core + ${assertj.version} + test + + + io.rest-assured + rest-assured + ${rest-assured.version} + test + + + org.hamcrest + hamcrest-all + ${hamcrest.version} + test + org.awaitility awaitility diff --git a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java index fabc35bc49..ec1ead3e6b 100644 --- a/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java +++ b/rest-client/src/main/java/org/thingsboard/rest/client/RestClient.java @@ -136,6 +136,8 @@ import org.thingsboard.server.common.data.rule.RuleChainMetaData; import org.thingsboard.server.common.data.rule.RuleChainType; import org.thingsboard.server.common.data.security.DeviceCredentials; import org.thingsboard.server.common.data.security.DeviceCredentialsType; +import org.thingsboard.server.common.data.security.model.JwtPair; +import org.thingsboard.server.common.data.security.model.JwtSettings; import org.thingsboard.server.common.data.security.model.SecuritySettings; import org.thingsboard.server.common.data.security.model.UserPasswordPolicy; import org.thingsboard.server.common.data.sms.config.TestSmsRequest; @@ -286,6 +288,23 @@ public class RestClient implements ClientHttpRequestInterceptor, Closeable { return restTemplate.postForEntity(baseURL + "/api/admin/securitySettings", securitySettings, SecuritySettings.class).getBody(); } + public Optional getJwtSettings() { + try { + ResponseEntity jwtSettings = restTemplate.getForEntity(baseURL + "/api/admin/jwtSettings", JwtSettings.class); + return Optional.ofNullable(jwtSettings.getBody()); + } catch (HttpClientErrorException exception) { + if (exception.getStatusCode() == HttpStatus.NOT_FOUND) { + return Optional.empty(); + } else { + throw exception; + } + } + } + + public JwtPair saveJwtSettings(JwtSettings jwtSettings) { + return restTemplate.postForEntity(baseURL + "/api/admin/jwtSettings", jwtSettings, JwtPair.class).getBody(); + } + public Optional getRepositorySettings() { try { ResponseEntity repositorySettings = restTemplate.getForEntity(baseURL + "/api/admin/repositorySettings", RepositorySettings.class); diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbFetchDeviceCredentialsNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbFetchDeviceCredentialsNode.java index c93c70ff07..26cff6a96d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbFetchDeviceCredentialsNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/metadata/TbFetchDeviceCredentialsNode.java @@ -30,6 +30,7 @@ import org.thingsboard.server.common.data.id.DeviceId; import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.plugin.ComponentType; import org.thingsboard.server.common.data.security.DeviceCredentials; +import org.thingsboard.server.common.data.security.DeviceCredentialsType; import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; @@ -78,16 +79,20 @@ public class TbFetchDeviceCredentialsNode implements TbNode { } TbMsg transformedMsg; - String credentialsType = deviceCredentials.getCredentialsType().name(); + DeviceCredentialsType credentialsType = deviceCredentials.getCredentialsType(); JsonNode credentialsInfo = ctx.getDeviceCredentialsService().toCredentialsInfo(deviceCredentials); if (fetchToMetadata) { TbMsgMetaData metaData = msg.getMetaData(); - metaData.putValue(CREDENTIALS_TYPE, credentialsType); - metaData.putValue(CREDENTIALS, JacksonUtil.toString(credentialsInfo)); + metaData.putValue(CREDENTIALS_TYPE, credentialsType.name()); + if (credentialsType.equals(DeviceCredentialsType.ACCESS_TOKEN) || credentialsType.equals(DeviceCredentialsType.X509_CERTIFICATE)) { + metaData.putValue(CREDENTIALS, credentialsInfo.asText()); + } else { + metaData.putValue(CREDENTIALS, JacksonUtil.toString(credentialsInfo)); + } transformedMsg = TbMsg.transformMsg(msg, msg.getType(), originator, metaData, msg.getData()); } else { ObjectNode data = (ObjectNode) JacksonUtil.toJsonNode(msg.getData()); - data.put(CREDENTIALS_TYPE, credentialsType); + data.put(CREDENTIALS_TYPE, credentialsType.name()); data.set(CREDENTIALS, credentialsInfo); transformedMsg = TbMsg.transformMsg(msg, msg.getType(), originator, msg.getMetaData(), JacksonUtil.toString(data)); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java index 58bb5c7564..b7414c5e0d 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/rest/TbHttpClient.java @@ -41,6 +41,7 @@ import org.springframework.util.concurrent.ListenableFutureCallback; import org.springframework.web.client.AsyncRestTemplate; import org.springframework.web.client.RestClientResponseException; import org.springframework.web.util.UriComponentsBuilder; +import org.thingsboard.common.util.JacksonUtil; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.rule.engine.api.TbNodeException; import org.thingsboard.rule.engine.api.TbRelationTypes; @@ -59,8 +60,11 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.security.NoSuchAlgorithmException; import java.util.Deque; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentLinkedDeque; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; @Data @Slf4j @@ -244,17 +248,33 @@ public class TbHttpClient { metaData.putValue(STATUS, response.getStatusCode().name()); metaData.putValue(STATUS_CODE, response.getStatusCode().value() + ""); metaData.putValue(STATUS_REASON, response.getStatusCode().getReasonPhrase()); - response.getHeaders().toSingleValueMap().forEach(metaData::putValue); + headersToMetaData(response.getHeaders(), metaData::putValue); String body = response.getBody() == null ? "{}" : response.getBody(); return ctx.transformMsg(origMsg, origMsg.getType(), origMsg.getOriginator(), metaData, body); } + void headersToMetaData(Map> headers, BiConsumer consumer) { + if (headers == null) { + return; + } + headers.forEach((key, values) -> { + if (values != null && !values.isEmpty()) { + if (values.size() == 1) { + consumer.accept(key, values.get(0)); + } else { + consumer.accept(key, JacksonUtil.toString(values)); + } + } + }); + } + private TbMsg processFailureResponse(TbContext ctx, TbMsg origMsg, ResponseEntity response) { TbMsgMetaData metaData = origMsg.getMetaData(); metaData.putValue(STATUS, response.getStatusCode().name()); metaData.putValue(STATUS_CODE, response.getStatusCode().value() + ""); metaData.putValue(STATUS_REASON, response.getStatusCode().getReasonPhrase()); metaData.putValue(ERROR_BODY, response.getBody()); + headersToMetaData(response.getHeaders(), metaData::putValue); return ctx.transformMsg(origMsg, origMsg.getType(), origMsg.getOriginator(), metaData, origMsg.getData()); } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java index 8a801cbb7f..52fa68dba8 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/transform/TbChangeOriginatorNode.java @@ -127,6 +127,7 @@ public class TbChangeOriginatorNode extends TbAbstractTransformNode { log.error("EntityNamePattern not specified for type [{}]", conf.getEntityType()); throw new IllegalArgumentException("Wrong config for [{}] in TbChangeOriginatorNode!" + ENTITY_SOURCE); } + EntitiesByNameAndTypeLoader.checkEntityType(EntityType.valueOf(conf.getEntityType())); } } diff --git a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java index d70865f197..21f2cf5822 100644 --- a/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java +++ b/rule-engine/rule-engine-components/src/main/java/org/thingsboard/rule/engine/util/EntitiesByNameAndTypeLoader.java @@ -16,75 +16,52 @@ package org.thingsboard.rule.engine.util; import org.thingsboard.rule.engine.api.TbContext; -import org.thingsboard.server.common.data.Customer; -import org.thingsboard.server.common.data.DashboardInfo; -import org.thingsboard.server.common.data.Device; import org.thingsboard.server.common.data.EntityType; -import org.thingsboard.server.common.data.EntityView; -import org.thingsboard.server.common.data.User; -import org.thingsboard.server.common.data.asset.Asset; -import org.thingsboard.server.common.data.edge.Edge; +import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.id.EntityId; -import java.util.Optional; +import java.util.List; public class EntitiesByNameAndTypeLoader { + private static final List AVAILABLE_ENTITY_TYPES = List.of( + EntityType.DEVICE, + EntityType.ASSET, + EntityType.ENTITY_VIEW, + EntityType.EDGE, + EntityType.USER); + public static EntityId findEntityId(TbContext ctx, EntityType entityType, String entityName) { - EntityId targetEntityId = null; + SearchTextBasedWithAdditionalInfo targetEntity; switch (entityType) { case DEVICE: - Device device = ctx.getDeviceService().findDeviceByTenantIdAndName(ctx.getTenantId(), entityName); - if (device != null) { - targetEntityId = device.getId(); - } + targetEntity = ctx.getDeviceService().findDeviceByTenantIdAndName(ctx.getTenantId(), entityName); break; case ASSET: - Asset asset = ctx.getAssetService().findAssetByTenantIdAndName(ctx.getTenantId(), entityName); - if (asset != null) { - targetEntityId = asset.getId(); - } - break; - case CUSTOMER: - Optional customerOptional = ctx.getCustomerService().findCustomerByTenantIdAndTitle(ctx.getTenantId(), entityName); - if (customerOptional.isPresent()) { - targetEntityId = customerOptional.get().getId(); - } - break; - case TENANT: - targetEntityId = ctx.getTenantId(); + targetEntity = ctx.getAssetService().findAssetByTenantIdAndName(ctx.getTenantId(), entityName); break; case ENTITY_VIEW: - EntityView entityView = ctx.getEntityViewService().findEntityViewByTenantIdAndName(ctx.getTenantId(), entityName); - if (entityView != null) { - targetEntityId = entityView.getId(); - } + targetEntity = ctx.getEntityViewService().findEntityViewByTenantIdAndName(ctx.getTenantId(), entityName); break; case EDGE: - Edge edge = ctx.getEdgeService().findEdgeByTenantIdAndName(ctx.getTenantId(), entityName); - if (edge != null) { - targetEntityId = edge.getId(); - } - break; - case DASHBOARD: - DashboardInfo dashboardInfo = ctx.getDashboardService().findFirstDashboardInfoByTenantIdAndName(ctx.getTenantId(), entityName); - if (dashboardInfo != null) { - targetEntityId = dashboardInfo.getId(); - } + targetEntity = ctx.getEdgeService().findEdgeByTenantIdAndName(ctx.getTenantId(), entityName); break; case USER: - User user = ctx.getUserService().findUserByEmail(ctx.getTenantId(), entityName); - if (user != null) { - targetEntityId = user.getId(); - } + targetEntity = ctx.getUserService().findUserByTenantIdAndEmail(ctx.getTenantId(), entityName); break; default: throw new IllegalStateException("Unexpected entity type " + entityType.name()); } - if (targetEntityId == null) { + if (targetEntity == null) { throw new IllegalStateException("Failed to found " + entityType.name() + " entity by name: '" + entityName + "'!"); } - return targetEntityId; + return targetEntity.getId(); + } + + public static void checkEntityType(EntityType entityType) { + if (!AVAILABLE_ENTITY_TYPES.contains(entityType)) { + throw new IllegalStateException("Unexpected entity type " + entityType.name()); + } } } diff --git a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java index f7a1d83ac9..28918538e2 100644 --- a/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java +++ b/rule-engine/rule-engine-components/src/test/java/org/thingsboard/rule/engine/rest/TbHttpClientTest.java @@ -18,6 +18,7 @@ package org.thingsboard.rule.engine.rest; import io.netty.channel.EventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; +import org.assertj.core.api.Assertions; import org.awaitility.Awaitility; import org.junit.After; import org.junit.Assert; @@ -26,6 +27,7 @@ import org.junit.Test; import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import org.mockserver.integration.ClientAndServer; +import org.springframework.util.LinkedMultiValueMap; import org.springframework.web.client.AsyncRestTemplate; import org.thingsboard.rule.engine.api.TbContext; import org.thingsboard.server.common.data.id.DeviceId; @@ -34,6 +36,8 @@ import org.thingsboard.server.common.msg.TbMsg; import org.thingsboard.server.common.msg.TbMsgMetaData; import java.net.URI; +import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -200,5 +204,23 @@ public class TbHttpClientTest { ); } + @Test + public void testHeadersToMetaData() { + Map> headers = new LinkedMultiValueMap<>(); + headers.put("Content-Type", List.of("binary")); + headers.put("Set-Cookie", List.of("sap-context=sap-client=075; path=/", "sap-token=sap-client=075; path=/")); + + TbMsgMetaData metaData = new TbMsgMetaData(); + + willCallRealMethod().given(client).headersToMetaData(any(), any()); + + client.headersToMetaData(headers, metaData::putValue); + + Map data = metaData.getData(); + + Assertions.assertThat(data).hasSize(2); + Assertions.assertThat(data.get("Content-Type")).isEqualTo("binary"); + Assertions.assertThat(data.get("Set-Cookie")).isEqualTo("[\"sap-context=sap-client=075; path=/\",\"sap-token=sap-client=075; path=/\"]"); + } } \ No newline at end of file diff --git a/ui-ngx/src/app/core/api/alarm-data-subscription.ts b/ui-ngx/src/app/core/api/alarm-data-subscription.ts index 42f915a12a..be134f81ab 100644 --- a/ui-ngx/src/app/core/api/alarm-data-subscription.ts +++ b/ui-ngx/src/app/core/api/alarm-data-subscription.ts @@ -62,6 +62,7 @@ export class AlarmDataSubscription { private alarmDataCommand: AlarmDataCmd; private pageData: PageData; + private prematureUpdates: Array>; private alarmIdToDataIndex: {[id: string]: number}; private subsTw: SubscriptionTimewindow; @@ -136,8 +137,21 @@ export class AlarmDataSubscription { this.subscriber.alarmData$.subscribe((alarmDataUpdate) => { if (alarmDataUpdate.data) { this.onPageData(alarmDataUpdate.data, alarmDataUpdate.allowedEntities, alarmDataUpdate.totalEntities); + if (this.prematureUpdates) { + for (const update of this.prematureUpdates) { + this.onDataUpdate(update); + } + this.prematureUpdates = null; + } } else if (alarmDataUpdate.update) { - this.onDataUpdate(alarmDataUpdate.update); + if (!this.pageData) { + if (!this.prematureUpdates) { + this.prematureUpdates = []; + } + this.prematureUpdates.push(alarmDataUpdate.update); + } else { + this.onDataUpdate(alarmDataUpdate.update); + } } }); diff --git a/ui-ngx/src/app/core/api/entity-data-subscription.ts b/ui-ngx/src/app/core/api/entity-data-subscription.ts index 3d176fc841..dfba1abaf2 100644 --- a/ui-ngx/src/app/core/api/entity-data-subscription.ts +++ b/ui-ngx/src/app/core/api/entity-data-subscription.ts @@ -120,6 +120,7 @@ export class EntityDataSubscription { private entityDataResolveSubject: Subject; private pageData: PageData; + private prematureUpdates: Array>; private data: Array>; private subsTw: SubscriptionTimewindow; private latestTsOffset: number; @@ -348,8 +349,21 @@ export class EntityDataSubscription { (entityDataUpdate) => { if (entityDataUpdate.data) { this.onPageData(entityDataUpdate.data); + if (this.prematureUpdates) { + for (const update of this.prematureUpdates) { + this.onDataUpdate(update); + } + this.prematureUpdates = null; + } } else if (entityDataUpdate.update) { - this.onDataUpdate(entityDataUpdate.update); + if (!this.pageData) { + if (!this.prematureUpdates) { + this.prematureUpdates = []; + } + this.prematureUpdates.push(entityDataUpdate.update); + } else { + this.onDataUpdate(entityDataUpdate.update); + } } } ); diff --git a/ui-ngx/src/app/core/auth/auth.service.ts b/ui-ngx/src/app/core/auth/auth.service.ts index 2b54306e2c..f5d2823508 100644 --- a/ui-ngx/src/app/core/auth/auth.service.ts +++ b/ui-ngx/src/app/core/auth/auth.service.ts @@ -195,18 +195,22 @@ export class AuthService { )); } - public logout(captureLastUrl: boolean = false) { + public logout(captureLastUrl: boolean = false, ignoreRequest = false) { if (captureLastUrl) { this.redirectUrl = this.router.url; } - this.http.post('/api/auth/logout', null, defaultHttpOptions(true, true)) - .subscribe(() => { - this.clearJwtToken(); - }, - () => { - this.clearJwtToken(); - } - ); + if (!ignoreRequest) { + this.http.post('/api/auth/logout', null, defaultHttpOptions(true, true)) + .subscribe(() => { + this.clearJwtToken(); + }, + () => { + this.clearJwtToken(); + } + ); + } else { + this.clearJwtToken(); + } } private notifyUserLoaded(isUserLoaded: boolean) { diff --git a/ui-ngx/src/app/core/http/admin.service.ts b/ui-ngx/src/app/core/http/admin.service.ts index 485f355b87..8933f7b99d 100644 --- a/ui-ngx/src/app/core/http/admin.service.ts +++ b/ui-ngx/src/app/core/http/admin.service.ts @@ -20,16 +20,18 @@ import { Observable } from 'rxjs'; import { HttpClient } from '@angular/common/http'; import { AdminSettings, - RepositorySettings, + AutoCommitSettings, + JwtSettings, MailServerSettings, + RepositorySettings, + RepositorySettingsInfo, SecuritySettings, TestSmsRequest, - UpdateMessage, - AutoCommitSettings, - RepositorySettingsInfo + UpdateMessage } from '@shared/models/settings.models'; import { EntitiesVersionControlService } from '@core/http/entities-version-control.service'; import { tap } from 'rxjs/operators'; +import { LoginResponse } from '@shared/models/login.models'; @Injectable({ providedIn: 'root' @@ -70,6 +72,14 @@ export class AdminService { defaultHttpOptionsFromConfig(config)); } + public getJwtSettings(config?: RequestConfig): Observable { + return this.http.get(`/api/admin/jwtSettings`, defaultHttpOptionsFromConfig(config)); + } + + public saveJwtSettings(jwtSettings: JwtSettings, config?: RequestConfig): Observable { + return this.http.post('/api/admin/jwtSettings', jwtSettings, defaultHttpOptionsFromConfig(config)); + } + public getRepositorySettings(config?: RequestConfig): Observable { return this.http.get(`/api/admin/repositorySettings`, defaultHttpOptionsFromConfig(config)); } diff --git a/ui-ngx/src/app/core/http/attribute.service.ts b/ui-ngx/src/app/core/http/attribute.service.ts index bad6547b5d..984b1e607b 100644 --- a/ui-ngx/src/app/core/http/attribute.service.ts +++ b/ui-ngx/src/app/core/http/attribute.service.ts @@ -43,7 +43,7 @@ export class AttributeService { public deleteEntityAttributes(entityId: EntityId, attributeScope: AttributeScope, attributes: Array, config?: RequestConfig): Observable { - const keys = attributes.map(attribute => encodeURI(attribute.key)).join(','); + const keys = attributes.map(attribute => encodeURIComponent(attribute.key)).join(','); return this.http.delete(`/api/plugins/telemetry/${entityId.entityType}/${entityId.id}/${attributeScope}` + `?keys=${keys}`, defaultHttpOptionsFromConfig(config)); @@ -51,7 +51,7 @@ export class AttributeService { public deleteEntityTimeseries(entityId: EntityId, timeseries: Array, deleteAllDataForKeys = false, startTs?: number, endTs?: number, config?: RequestConfig): Observable { - const keys = timeseries.map(attribute => encodeURI(attribute.key)).join(','); + const keys = timeseries.map(attribute => encodeURIComponent(attribute.key)).join(','); let url = `/api/plugins/telemetry/${entityId.entityType}/${entityId.id}/timeseries/delete` + `?keys=${keys}&deleteAllDataForKeys=${deleteAllDataForKeys}`; if (isDefinedAndNotNull(startTs)) { diff --git a/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts b/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts index 15c4d3ede3..bb9a7cef1d 100644 --- a/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts +++ b/ui-ngx/src/app/core/interceptors/global-http-interceptor.ts @@ -103,7 +103,8 @@ export class GlobalHttpInterceptor implements HttpInterceptor { const ignoreErrors = config.ignoreErrors; const resendRequest = config.resendRequest; const errorCode = errorResponse.error ? errorResponse.error.errorCode : null; - if (errorResponse.error && errorResponse.error.refreshTokenPending || errorResponse.status === 401) { + if (errorResponse.error && errorResponse.error.refreshTokenPending || + errorResponse.status === 401 && req.url !== Constants.entryPoints.tokenRefresh) { if (errorResponse.error && errorResponse.error.refreshTokenPending || errorCode && errorCode === Constants.serverErrorCode.jwtTokenExpired) { return this.refreshTokenAndRetry(req, next); @@ -153,7 +154,7 @@ export class GlobalHttpInterceptor implements HttpInterceptor { return this.jwtIntercept(req, next); }), catchError((err: Error) => { - this.authService.logout(true); + this.authService.logout(true, true); const message = err ? err.message : 'Unauthorized!'; return this.handleResponseError(req, next, new HttpErrorResponse({error: {message, timeout: 200}, status: 401})); })); diff --git a/ui-ngx/src/app/core/ws/telemetry-websocket.service.ts b/ui-ngx/src/app/core/ws/telemetry-websocket.service.ts index 783b56f844..92dccc0d88 100644 --- a/ui-ngx/src/app/core/ws/telemetry-websocket.service.ts +++ b/ui-ngx/src/app/core/ws/telemetry-websocket.service.ts @@ -233,7 +233,7 @@ export class TelemetryWebsocketService implements TelemetryService { }, () => { this.isOpening = false; - this.authService.logout(true); + this.authService.logout(true, true); } ); } diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html index 80c9f384fd..39db84d8c6 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.html @@ -15,144 +15,239 @@ limitations under the License. --> -
- - -
- admin.security-settings - -
-
- - - -
- -
-
-
- - - - -
admin.general-policy
- - - - admin.max-failed-login-attempts - - - {{ 'admin.minimum-max-failed-login-attempts-range' | translate }} - - - - admin.user-lockout-notification-email - - - - - - -
admin.password-policy
- - -
- - admin.minimum-password-length - - - {{ 'admin.minimum-password-length-required' | translate }} - - - {{ 'admin.minimum-password-length-range' | translate }} - - - {{ 'admin.minimum-password-length-range' | translate }} - - - - admin.minimum-uppercase-letters - - - {{ 'admin.minimum-uppercase-letters-range' | translate }} - - - - admin.minimum-lowercase-letters - - - {{ 'admin.minimum-lowercase-letters-range' | translate }} - - - - admin.minimum-digits - - - {{ 'admin.minimum-digits-range' | translate }} - - - - admin.minimum-special-characters - - - {{ 'admin.minimum-special-characters-range' | translate }} - - - - admin.password-expiration-period-days - - - {{ 'admin.password-expiration-period-days-range' | translate }} - - - - admin.password-reuse-frequency-days - - - {{ 'admin.password-reuse-frequency-days-range' | translate }} - - - - admin.allow-whitespace - -
- - -
-
- -
+ + +
+ admin.security-settings + +
+
+ + + +
+ + +
+
+ admin.general-policy + + admin.max-failed-login-attempts + + + {{ 'admin.minimum-max-failed-login-attempts-range' | translate }} + + + + admin.user-lockout-notification-email + + +
+ +
+ admin.password-policy +
+ + admin.minimum-password-length + + + {{ 'admin.minimum-password-length-required' | translate }} + + + {{ 'admin.minimum-password-length-range' | translate }} + + + {{ 'admin.minimum-password-length-range' | translate }} + + +
+ + admin.minimum-uppercase-letters + + + {{ 'admin.minimum-uppercase-letters-range' | translate }} + + + + admin.minimum-lowercase-letters + + + {{ 'admin.minimum-lowercase-letters-range' | translate }} + + +
+
+ + admin.minimum-digits + + + {{ 'admin.minimum-digits-range' | translate }} + + + + admin.minimum-special-characters + + + {{ 'admin.minimum-special-characters-range' | translate }} + + +
+
+ + admin.password-expiration-period-days + + + {{ 'admin.password-expiration-period-days-range' | translate }} + + + + admin.password-reuse-frequency-days + + + {{ 'admin.password-reuse-frequency-days-range' | translate }} + + +
+ + admin.allow-whitespace + +
- - - -
+
+ + +
+ + + + + + +
+ admin.jwt.security-settings +
+ + +
+
+
+ + admin.jwt.issuer-name + + + {{ 'admin.jwt.issuer-name-required' | translate }} + + + + admin.jwt.signings-key + + + admin.jwt.signings-key-hint + + {{ 'admin.jwt.signings-key-required' | translate }} + + + {{ 'admin.jwt.signings-key-base64' | translate }} + + + {{ 'admin.jwt.signings-key-min-length' | translate }} + + +
+
+ + admin.jwt.expiration-time + + + {{ 'admin.jwt.expiration-time-required' | translate }} + + + {{ 'admin.jwt.expiration-time-pattern' | translate }} + + + {{ 'admin.jwt.expiration-time-min' | translate }} + + + + admin.jwt.refresh-expiration-time + + + {{ 'admin.jwt.refresh-expiration-time-required' | translate }} + + + {{ 'admin.jwt.refresh-expiration-time-pattern' | translate }} + + + {{ 'admin.jwt.refresh-expiration-time-min' | translate }} + + + {{ 'admin.jwt.refresh-expiration-time-less-token' | translate }} + + +
+
+ + +
+
+
+ + diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss index 32e514e010..5a6d5eed04 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.scss @@ -14,7 +14,26 @@ * limitations under the License. */ :host { - .mat-accordion-container { - margin-bottom: 16px; + .mat-headline { + margin-bottom: 8px; + } + + .mat-card-title { + margin: 0; + } + + .mat-card-content { + padding: 0 !important; + } + + .fields-group { + padding: 8px 16px 0; + margin: 10px 0; + border: 1px groove rgba(0, 0, 0, .25); + border-radius: 4px; + + legend { + color: rgba(0, 0, 0, .7); + } } } diff --git a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts index 6dc070b278..628d23e78a 100644 --- a/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts +++ b/ui-ngx/src/app/modules/home/pages/admin/security-settings.component.ts @@ -14,40 +14,50 @@ /// limitations under the License. /// -import { Component, OnInit } from '@angular/core'; +import { Component } from '@angular/core'; import { Store } from '@ngrx/store'; import { AppState } from '@core/core.state'; import { PageComponent } from '@shared/components/page.component'; import { Router } from '@angular/router'; -import { FormBuilder, FormGroup, Validators } from '@angular/forms'; -import { SecuritySettings } from '@shared/models/settings.models'; +import { FormBuilder, FormControl, FormGroup, Validators } from '@angular/forms'; +import { JwtSettings, SecuritySettings } from '@shared/models/settings.models'; import { AdminService } from '@core/http/admin.service'; import { HasConfirmForm } from '@core/guards/confirm-on-exit.guard'; +import { mergeMap, tap } from 'rxjs/operators'; +import { randomAlphanumeric } from '@core/utils'; +import { AuthService } from '@core/auth/auth.service'; +import { DialogService } from '@core/services/dialog.service'; +import { TranslateService } from '@ngx-translate/core'; +import { Observable, of } from 'rxjs'; @Component({ selector: 'tb-security-settings', templateUrl: './security-settings.component.html', styleUrls: ['./security-settings.component.scss', './settings-card.scss'] }) -export class SecuritySettingsComponent extends PageComponent implements OnInit, HasConfirmForm { +export class SecuritySettingsComponent extends PageComponent implements HasConfirmForm { securitySettingsFormGroup: FormGroup; - securitySettings: SecuritySettings; + jwtSecuritySettingsFormGroup: FormGroup; + + private securitySettings: SecuritySettings; + private jwtSettings: JwtSettings; constructor(protected store: Store, private router: Router, private adminService: AdminService, - public fb: FormBuilder) { + private authService: AuthService, + private dialogService: DialogService, + private translate: TranslateService, + private fb: FormBuilder) { super(store); - } - - ngOnInit() { this.buildSecuritySettingsForm(); + this.buildJwtSecuritySettingsForm(); this.adminService.getSecuritySettings().subscribe( - (securitySettings) => { - this.securitySettings = securitySettings; - this.securitySettingsFormGroup.reset(this.securitySettings); - } + securitySettings => this.processSecuritySettings(securitySettings) + ); + this.adminService.getJwtSettings().subscribe( + jwtSettings => this.processJwtSettings(jwtSettings) ); } @@ -70,18 +80,114 @@ export class SecuritySettingsComponent extends PageComponent implements OnInit, }); } + buildJwtSecuritySettingsForm() { + this.jwtSecuritySettingsFormGroup = this.fb.group({ + tokenIssuer: ['', Validators.required], + tokenSigningKey: ['', [Validators.required, this.base64Format]], + tokenExpirationTime: [0, [Validators.required, Validators.pattern('[0-9]*'), Validators.min(60)]], + refreshTokenExpTime: [0, [Validators.required, Validators.pattern('[0-9]*'), Validators.min(900)]] + }, {validators: this.refreshTokenTimeGreatTokenTime.bind(this)}); + this.jwtSecuritySettingsFormGroup.get('tokenExpirationTime').valueChanges.subscribe( + () => this.jwtSecuritySettingsFormGroup.get('refreshTokenExpTime').updateValueAndValidity({onlySelf: true}) + ); + } + save(): void { this.securitySettings = {...this.securitySettings, ...this.securitySettingsFormGroup.value}; this.adminService.saveSecuritySettings(this.securitySettings).subscribe( - (securitySettings) => { - this.securitySettings = securitySettings; - this.securitySettingsFormGroup.reset(this.securitySettings); - } + securitySettings => this.processSecuritySettings(securitySettings) ); } + saveJwtSettings() { + const jwtFormSettings = this.jwtSecuritySettingsFormGroup.value; + this.confirmChangeJWTSettings().pipe(mergeMap(value => { + if (value) { + return this.adminService.saveJwtSettings(jwtFormSettings).pipe( + tap((data) => this.authService.setUserFromJwtToken(data.token, data.refreshToken, false)), + mergeMap(() => this.adminService.getJwtSettings()), + tap(jwtSettings => this.processJwtSettings(jwtSettings)) + ); + } + return of(null); + })).subscribe(() => {}); + } + + discardSetting() { + this.securitySettingsFormGroup.reset(this.securitySettings); + } + + discardJwtSetting() { + this.jwtSecuritySettingsFormGroup.reset(this.jwtSettings); + } + + markAsTouched() { + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsTouched(); + } + + private confirmChangeJWTSettings(): Observable { + if (this.jwtSecuritySettingsFormGroup.get('tokenIssuer').value !== (this.jwtSettings?.tokenIssuer || '') || + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').value !== (this.jwtSettings?.tokenSigningKey || '')) { + return this.dialogService.confirm( + this.translate.instant('admin.jwt.info-header'), + `
${this.translate.instant('admin.jwt.info-message')}
`, + this.translate.instant('action.discard-changes'), + this.translate.instant('action.confirm') + ); + } + return of(true); + } + + generateSigningKey() { + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').setValue(btoa(randomAlphanumeric(64))); + if (this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').pristine) { + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsDirty(); + this.jwtSecuritySettingsFormGroup.get('tokenSigningKey').markAsTouched(); + } + } + + private processSecuritySettings(securitySettings: SecuritySettings) { + this.securitySettings = securitySettings; + this.securitySettingsFormGroup.reset(this.securitySettings); + } + + private processJwtSettings(jwtSettings: JwtSettings) { + this.jwtSettings = jwtSettings; + this.jwtSecuritySettingsFormGroup.reset(jwtSettings); + } + + private refreshTokenTimeGreatTokenTime(formGroup: FormGroup): { [key: string]: boolean } | null { + if (formGroup) { + const tokenTime = formGroup.value.tokenExpirationTime; + const refreshTokenTime = formGroup.value.refreshTokenExpTime; + if (tokenTime >= refreshTokenTime ) { + if (formGroup.get('refreshTokenExpTime').untouched) { + formGroup.get('refreshTokenExpTime').markAsTouched(); + } + formGroup.get('refreshTokenExpTime').setErrors({lessToken: true}); + return {lessToken: true}; + } + } + return null; + } + + private base64Format(control: FormControl): { [key: string]: boolean } | null { + if (control.value === '' || control.value === 'thingsboardDefaultSigningKey') { + return null; + } + try { + const value = atob(control.value); + if (value.length < 32) { + return {minLength: true}; + } + return null; + } catch (e) { + return {base64: true}; + } + } + confirmForm(): FormGroup { - return this.securitySettingsFormGroup; + return this.securitySettingsFormGroup.dirty ? this.securitySettingsFormGroup : this.jwtSecuritySettingsFormGroup; } } diff --git a/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html b/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html index 710339908b..f7586d5656 100644 --- a/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html +++ b/ui-ngx/src/app/shared/components/dialog/confirm-dialog.component.html @@ -16,7 +16,7 @@ -->

{{data.title}}

-
+
diff --git a/ui-ngx/src/app/shared/models/settings.models.ts b/ui-ngx/src/app/shared/models/settings.models.ts index 83426c8b3b..8ecb4066f1 100644 --- a/ui-ngx/src/app/shared/models/settings.models.ts +++ b/ui-ngx/src/app/shared/models/settings.models.ts @@ -63,6 +63,13 @@ export interface SecuritySettings { passwordPolicy: UserPasswordPolicy; } +export interface JwtSettings { + tokenIssuer: string; + tokenSigningKey: string; + tokenExpirationTime: number; + refreshTokenExpTime: number; +} + export interface UpdateMessage { message: string; updateAvailable: boolean; diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 3f3f70860f..511d9def5d 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -381,6 +381,28 @@ "within-time": "Within time (sec)", "within-time-pattern": "Time must be a positive integer.", "within-time-required": "Time is required." + }, + "jwt": { + "security-settings": "JWT security settings", + "issuer-name": "Issuer name", + "issuer-name-required": "Issuer name is required.", + "signings-key": "Signing key", + "signings-key-hint": "Base64 encoded string representing at least 256 bits of data.", + "signings-key-required": "Signing key is required.", + "signings-key-min-length": "Signing key must be at least 256 bits of data.", + "signings-key-base64": "Signing key must be base64 format.", + "expiration-time": "Token expiration time (sec)", + "expiration-time-required": "Token expiration time is required.", + "expiration-time-pattern": "Token expiration time be a positive integer.", + "expiration-time-min": "Minimum time is 60 seconds (1 minute).", + "refresh-expiration-time": "Refresh token expiration time (sec)", + "refresh-expiration-time-required": "Refresh token expiration time is required.", + "refresh-expiration-time-pattern": "Refresh token expiration time be a positive integer.", + "refresh-expiration-time-min": "Minimum time is 900 seconds (15 minute).", + "refresh-expiration-time-less-token": "Refresh token time must be greater token time.", + "generate-key": "Generate key", + "info-header": "All users will be to re-logined", + "info-message": "Change of the JWT Signing Key will cause all issued tokens to be invalid. All users will need to re-login. This will also affect scripts that use Rest API/Websockets." } }, "alarm": {