From 204b6197f1233e42f114374d095c65d6f58f96d6 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Mon, 27 Jun 2022 14:44:55 +0300 Subject: [PATCH 1/9] Do not register import result on reimport --- .../sync/vc/DefaultEntitiesVersionControlService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 10bedc4b20..f3bd00ed41 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -382,12 +382,12 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont } catch (Exception e) { throw new LoadEntityException(entityData, e); } + registerResult(ctx, entityType, importResult); + if (!importResult.isUpdatedAllExternalIds()) { ctx.getToReimport().put(entityData.getEntity().getExternalId(), new ReimportTask(reimportBackup, ctx.getSettings())); continue; } - - registerResult(ctx, entityType, importResult); ctx.getImportedEntities().computeIfAbsent(entityType, t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } @@ -405,7 +405,6 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont ctx.setSettings(settings); EntityImportResult importResult = exportImportService.importEntity(ctx, entityData); - registerResult(ctx, externalId.getEntityType(), importResult); ctx.getImportedEntities().computeIfAbsent(externalId.getEntityType(), t -> new HashSet<>()) .add(importResult.getSavedEntity().getId()); } catch (Exception e) { From 055919c8a7efa1238deb2a48086e64f43c927d59 Mon Sep 17 00:00:00 2001 From: Volodymyr Babak Date: Tue, 28 Jun 2022 12:33:45 +0300 Subject: [PATCH 2/9] Saving of edge event in batches should be single threaded. persisten attributes should not be propagated to edge --- .../service/edge/rpc/sync/DefaultEdgeRequestsService.java | 4 ++++ application/src/main/resources/thingsboard.yml | 1 - .../thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java | 5 +---- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java index 210ad30c44..c42de1b68c 100644 --- a/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java +++ b/application/src/main/java/org/thingsboard/server/service/edge/rpc/sync/DefaultEdgeRequestsService.java @@ -75,6 +75,7 @@ import org.thingsboard.server.gen.edge.v1.UserCredentialsRequestMsg; import org.thingsboard.server.gen.edge.v1.WidgetBundleTypesRequestMsg; import org.thingsboard.server.service.entitiy.entityView.TbEntityViewService; import org.thingsboard.server.service.executors.DbCallbackExecutorService; +import org.thingsboard.server.service.state.DefaultDeviceStateService; import java.util.ArrayList; import java.util.HashMap; @@ -163,6 +164,9 @@ public class DefaultEdgeRequestsService implements EdgeRequestsService { Map entityData = new HashMap<>(); ObjectNode attributes = mapper.createObjectNode(); for (AttributeKvEntry attr : ssAttributes) { + if (DefaultDeviceStateService.PERSISTENT_ATTRIBUTES.contains(attr.getKey())) { + continue; + } if (attr.getDataType() == DataType.BOOLEAN && attr.getBooleanValue().isPresent()) { attributes.put(attr.getKey(), attr.getBooleanValue().get()); } else if (attr.getDataType() == DataType.DOUBLE && attr.getDoubleValue().isPresent()) { diff --git a/application/src/main/resources/thingsboard.yml b/application/src/main/resources/thingsboard.yml index f6312a7659..05cd5a2599 100644 --- a/application/src/main/resources/thingsboard.yml +++ b/application/src/main/resources/thingsboard.yml @@ -266,7 +266,6 @@ sql: batch_size: "${SQL_EDGE_EVENTS_BATCH_SIZE:1000}" batch_max_delay: "${SQL_EDGE_EVENTS_BATCH_MAX_DELAY_MS:100}" stats_print_interval_ms: "${SQL_EDGE_EVENTS_BATCH_STATS_PRINT_MS:10000}" - batch_threads: "${SQL_EDGE_EVENTS_BATCH_THREADS:3}" # batch thread count have to be a prime number like 3 or 5 to gain perfect hash distribution # Specify whether to sort entities before batch update. Should be enabled for cluster mode to avoid deadlocks batch_sort: "${SQL_BATCH_SORT:false}" # Specify whether to remove null characters from strValue of attributes and timeseries before insert diff --git a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java index 5ecb60e744..36f834b8bc 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java +++ b/dao/src/main/java/org/thingsboard/server/dao/sql/edge/JpaBaseEdgeEventDao.java @@ -72,9 +72,6 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao queue; @Autowired @@ -110,7 +107,7 @@ public class JpaBaseEdgeEventDao extends JpaAbstractSearchTextDao(params, hashcodeFunction, batchThreads, statsFactory); + queue = new TbSqlBlockingQueueWrapper<>(params, hashcodeFunction, 1, statsFactory); queue.init(logExecutor, v -> edgeEventInsertRepository.save(v), Comparator.comparing(EdgeEventEntity::getTs) ); From 4d1a82d50b2e21be39966cd7c66fde2d8ebc8c19 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 28 Jun 2022 13:39:09 +0300 Subject: [PATCH 3/9] Fix export/import of deprecated dashboards --- .../server/common/data/Dashboard.java | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java index 049c830745..f0d0d88f68 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/Dashboard.java @@ -17,19 +17,18 @@ package org.thingsboard.server.common.data; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Streams; import io.swagger.annotations.ApiModelProperty; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; import org.thingsboard.server.common.data.id.DashboardId; -import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.stream.StreamSupport; +import java.util.stream.Collectors; @EqualsAndHashCode(callSuper = true) public class Dashboard extends DashboardInfo implements ExportableEntity { @@ -86,17 +85,11 @@ public class Dashboard extends DashboardInfo implements ExportableEntity getChildObjects(String propertyName) { return Optional.ofNullable(configuration) .map(config -> config.get(propertyName)) - .filter(node -> !node.isEmpty()) - .map(node -> (ObjectNode) node) - .map(object -> { - List widgets = new ArrayList<>(object.size()); - object.forEach(child -> { - if (child.isObject()) { - widgets.add((ObjectNode) child); - } - }); - return widgets; - }) + .filter(node -> !node.isEmpty() && (node.isObject() || node.isArray())) + .map(node -> Streams.stream(node.elements()) + .filter(JsonNode::isObject) + .map(jsonNode -> (ObjectNode) jsonNode) + .collect(Collectors.toList())) .orElse(Collections.emptyList()); } From 80f1df46953cfb758d43a2f7f62afa43c90c79dc Mon Sep 17 00:00:00 2001 From: Igor Kulikov Date: Tue, 28 Jun 2022 14:44:33 +0300 Subject: [PATCH 4/9] Improve TB security configuration - remove deprecated configurations. --- .../ThingsboardSecurityConfiguration.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java index 76c631bddb..a6540f007a 100644 --- a/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java +++ b/application/src/main/java/org/thingsboard/server/config/ThingsboardSecurityConfiguration.java @@ -24,22 +24,22 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.annotation.Order; import org.springframework.security.authentication.AuthenticationManager; +import org.springframework.security.authentication.DefaultAuthenticationEventPublisher; +import org.springframework.security.config.annotation.ObjectPostProcessor; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.method.configuration.EnableGlobalMethodSecurity; import org.springframework.security.config.annotation.web.builders.HttpSecurity; -import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; import org.springframework.security.config.http.SessionCreationPolicy; import org.springframework.security.crypto.bcrypt.BCryptPasswordEncoder; import org.springframework.security.oauth2.client.web.OAuth2AuthorizationRequestResolver; +import org.springframework.security.web.SecurityFilterChain; import org.springframework.security.web.authentication.AuthenticationFailureHandler; import org.springframework.security.web.authentication.AuthenticationSuccessHandler; import org.springframework.security.web.authentication.UsernamePasswordAuthenticationFilter; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; import org.springframework.web.cors.UrlBasedCorsConfigurationSource; import org.springframework.web.filter.CorsFilter; -import org.thingsboard.server.dao.audit.AuditLogLevelFilter; import org.thingsboard.server.dao.oauth2.OAuth2Configuration; import org.thingsboard.server.exception.ThingsboardErrorResponseHandler; import org.thingsboard.server.queue.util.TbCoreComponent; @@ -63,7 +63,7 @@ import java.util.List; @EnableGlobalMethodSecurity(prePostEnabled=true) @Order(SecurityProperties.BASIC_AUTH_ORDER) @TbCoreComponent -public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapter { +public class ThingsboardSecurityConfiguration { public static final String JWT_TOKEN_HEADER_PARAM = "X-Authorization"; public static final String JWT_TOKEN_HEADER_PARAM_V2 = "Authorization"; @@ -161,16 +161,15 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt } @Bean - @Override - public AuthenticationManager authenticationManagerBean() throws Exception { - return super.authenticationManagerBean(); - } - - @Override - protected void configure(AuthenticationManagerBuilder auth) { + public AuthenticationManager authenticationManager(ObjectPostProcessor objectPostProcessor) throws Exception { + DefaultAuthenticationEventPublisher eventPublisher = objectPostProcessor + .postProcess(new DefaultAuthenticationEventPublisher()); + var auth = new AuthenticationManagerBuilder(objectPostProcessor); + auth.authenticationEventPublisher(eventPublisher); auth.authenticationProvider(restAuthenticationProvider); auth.authenticationProvider(jwtAuthenticationProvider); auth.authenticationProvider(refreshTokenAuthenticationProvider); + return auth.build(); } @Bean @@ -181,18 +180,20 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt @Autowired private OAuth2AuthorizationRequestResolver oAuth2AuthorizationRequestResolver; - @Override - public void configure(WebSecurity web) throws Exception { - web.ignoring().antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**"); + @Bean + @Order(0) + SecurityFilterChain resources(HttpSecurity http) throws Exception { + http + .requestMatchers((matchers) -> matchers.antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**")) + .authorizeHttpRequests((authorize) -> authorize.anyRequest().permitAll()) + .requestCache().disable() + .securityContext().disable() + .sessionManagement().disable(); + return http.build(); } - @Override - protected void configure(HttpSecurity http) throws Exception { -// http.authorizeHttpRequests((authorizeHttpRequests) -> -// authorizeHttpRequests -// .antMatchers("/*.js","/*.css","/*.ico","/assets/**","/static/**") -// .permitAll() -// ); + @Bean + SecurityFilterChain filterChain(HttpSecurity http) throws Exception { http.headers().cacheControl().and().frameOptions().disable() .and() .cors() @@ -234,6 +235,7 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt .successHandler(oauth2AuthenticationSuccessHandler) .failureHandler(oauth2AuthenticationFailureHandler); } + return http.build(); } @Bean @@ -247,5 +249,4 @@ public class ThingsboardSecurityConfiguration extends WebSecurityConfigurerAdapt return new CorsFilter(source); } } - } From 4f74df66cd0012086f0d86ddcc47b4061be12209 Mon Sep 17 00:00:00 2001 From: Viacheslav Klimov Date: Tue, 28 Jun 2022 15:25:36 +0300 Subject: [PATCH 5/9] Fix invalid error message on relations import failure --- .../ie/DefaultEntitiesExportImportService.java | 14 +++++++++++--- .../DefaultEntitiesVersionControlService.java | 18 +++++++++--------- .../service/sync/vc/LoadEntityException.java | 8 ++++---- .../sync/vc/data/EntitiesImportCtx.java | 6 +++--- 4 files changed, 27 insertions(+), 19 deletions(-) diff --git a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java index 2d6d14c4ac..0f43159e30 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/ie/DefaultEntitiesExportImportService.java @@ -38,6 +38,8 @@ import org.thingsboard.server.service.sync.ie.exporting.EntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.BaseEntityExportService; import org.thingsboard.server.service.sync.ie.exporting.impl.DefaultEntityExportService; import org.thingsboard.server.service.sync.ie.importing.EntityImportService; +import org.thingsboard.server.service.sync.ie.importing.impl.MissingEntityException; +import org.thingsboard.server.service.sync.vc.LoadEntityException; import org.thingsboard.server.service.sync.vc.data.EntitiesExportCtx; import org.thingsboard.server.service.sync.vc.data.EntitiesImportCtx; @@ -95,15 +97,21 @@ public class DefaultEntitiesExportImportService implements EntitiesExportImportS EntityImportResult importResult = importService.importEntity(ctx, exportData); ctx.putInternalId(exportData.getExternalId(), importResult.getSavedEntity().getId()); - ctx.addReferenceCallback(importResult.getSaveReferencesCallback()); + ctx.addReferenceCallback(exportData.getExternalId(), importResult.getSaveReferencesCallback()); ctx.addEventCallback(importResult.getSendEventsCallback()); return importResult; } @Override public void saveReferencesAndRelations(EntitiesImportCtx ctx) throws ThingsboardException { - for (ThrowingRunnable saveReferencesCallback : ctx.getReferenceCallbacks()) { - saveReferencesCallback.run(); + for (Map.Entry callbackEntry : ctx.getReferenceCallbacks().entrySet()) { + EntityId externalId = callbackEntry.getKey(); + ThrowingRunnable saveReferencesCallback = callbackEntry.getValue(); + try { + saveReferencesCallback.run(); + } catch (MissingEntityException e) { + throw new LoadEntityException(externalId, e); + } } relationService.saveRelations(ctx.getTenantId(), new ArrayList<>(ctx.getRelations())); diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java index 0f6eb1b2f3..88a15f7cdc 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/DefaultEntitiesVersionControlService.java @@ -286,7 +286,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont result.setDone(true); return cachePut(ctx.getRequestId(), result); } catch (LoadEntityException e) { - return cachePut(ctx.getRequestId(), onError(e.getData(), e.getCause())); + return cachePut(ctx.getRequestId(), onError(e.getExternalId(), e.getCause())); } catch (Exception e) { log.info("[{}] Failed to process request [{}] due to: ", ctx.getTenantId(), request, e); return cachePut(ctx.getRequestId(), VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage()))); @@ -313,7 +313,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont .deleted(0) .build()); } catch (Exception e) { - throw new LoadEntityException(entityData, e); + throw new LoadEntityException(entityData.getExternalId(), e); } } @@ -382,7 +382,7 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont try { importResult = exportImportService.importEntity(ctx, entityData); } catch (Exception e) { - throw new LoadEntityException(entityData, e); + throw new LoadEntityException(entityData.getExternalId(), e); } registerResult(ctx, entityType, importResult); @@ -432,20 +432,20 @@ public class DefaultEntitiesVersionControlService implements EntitiesVersionCont }); } - private VersionLoadResult onError(EntityExportData entityData, Throwable e) { - return analyze(e, entityData).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage()))); + private VersionLoadResult onError(EntityId externalId, Throwable e) { + return analyze(e, externalId).orElse(VersionLoadResult.error(EntityLoadError.runtimeError(e.getMessage()))); } - private Optional analyze(Throwable e, EntityExportData entityData) { + private Optional analyze(Throwable e, EntityId externalId) { if (e == null) { return Optional.empty(); } else { if (e instanceof DeviceCredentialsValidationException) { - return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(entityData.getExternalId()))); + return Optional.of(VersionLoadResult.error(EntityLoadError.credentialsError(externalId))); } else if (e instanceof MissingEntityException) { - return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(entityData.getExternalId(), ((MissingEntityException) e).getEntityId()))); + return Optional.of(VersionLoadResult.error(EntityLoadError.referenceEntityError(externalId, ((MissingEntityException) e).getEntityId()))); } else { - return analyze(e.getCause(), entityData); + return analyze(e.getCause(), externalId); } } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java index a1b036d37b..13d8280046 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/LoadEntityException.java @@ -16,17 +16,17 @@ package org.thingsboard.server.service.sync.vc; import lombok.Getter; -import org.thingsboard.server.common.data.sync.ie.EntityExportData; +import org.thingsboard.server.common.data.id.EntityId; @SuppressWarnings("rawtypes") public class LoadEntityException extends RuntimeException { private static final long serialVersionUID = -1749719992370409504L; @Getter - private final EntityExportData data; + private final EntityId externalId; - public LoadEntityException(EntityExportData data, Throwable cause) { + public LoadEntityException(EntityId externalId, Throwable cause) { super(cause); - this.data = data; + this.externalId = externalId; } } diff --git a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java index d7b85a1560..fddc91883e 100644 --- a/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java +++ b/application/src/main/java/org/thingsboard/server/service/sync/vc/data/EntitiesImportCtx.java @@ -48,7 +48,7 @@ public class EntitiesImportCtx { private final Map results = new HashMap<>(); private final Map> importedEntities = new HashMap<>(); private final Map toReimport = new HashMap<>(); - private final List referenceCallbacks = new ArrayList<>(); + private final Map referenceCallbacks = new HashMap<>(); private final List eventCallbacks = new ArrayList<>(); private final Map externalToInternalIdMap = new HashMap<>(); private final Set notFoundIds = new HashSet<>(); @@ -119,9 +119,9 @@ public class EntitiesImportCtx { relations.addAll(values); } - public void addReferenceCallback(ThrowingRunnable tr) { + public void addReferenceCallback(EntityId externalId, ThrowingRunnable tr) { if (tr != null) { - referenceCallbacks.add(tr); + referenceCallbacks.put(externalId, tr); } } From 942e6aaccf09d4b2a92f8bea9fef37ce8d244dcb Mon Sep 17 00:00:00 2001 From: Vladyslav_Prykhodko Date: Tue, 28 Jun 2022 15:30:28 +0300 Subject: [PATCH 6/9] UI: Fix change password current password not required --- .../app/modules/home/pages/security/security.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui-ngx/src/app/modules/home/pages/security/security.component.html b/ui-ngx/src/app/modules/home/pages/security/security.component.html index a79f2397b2..820ee69bdd 100644 --- a/ui-ngx/src/app/modules/home/pages/security/security.component.html +++ b/ui-ngx/src/app/modules/home/pages/security/security.component.html @@ -38,9 +38,9 @@

profile.change-password

- + profile.current-password - + {{ 'security.password-requirement.incorrect-password-try-again' | translate }} From 68f0388723f135972b764c9c62e7ea2512370425 Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Jun 2022 14:54:38 +0200 Subject: [PATCH 7/9] added queue name and topic validation --- .../server/common/data/queue/Queue.java | 6 ++++++ .../dao/service/validator/QueueValidator.java | 14 ++++++++++++-- .../validator/TenantProfileDataValidator.java | 13 +++++++++++-- .../components/queue/queue-form.component.html | 3 +++ .../home/components/queue/queue-form.component.ts | 2 +- .../src/assets/locale/locale.constant-en_US.json | 1 + 6 files changed, 34 insertions(+), 5 deletions(-) diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java index b4d65706a2..a6623be8a4 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/queue/Queue.java @@ -22,11 +22,17 @@ import org.thingsboard.server.common.data.SearchTextBasedWithAdditionalInfo; import org.thingsboard.server.common.data.id.QueueId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.tenant.profile.TenantProfileQueueConfiguration; +import org.thingsboard.server.common.data.validation.Length; +import org.thingsboard.server.common.data.validation.NoXss; @Data public class Queue extends SearchTextBasedWithAdditionalInfo implements HasName, HasTenantId { private TenantId tenantId; + @NoXss + @Length(fieldName = "name") private String name; + @NoXss + @Length(fieldName = "topic") private String topic; private int pollInterval; private int partitions; diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java index c8ce639d40..a1d0df43d5 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java @@ -29,6 +29,8 @@ import org.thingsboard.server.dao.queue.QueueDao; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; +import java.util.regex.Pattern; + @Component public class QueueValidator extends DataValidator { @@ -38,6 +40,8 @@ public class QueueValidator extends DataValidator { @Autowired private TbTenantProfileCache tenantProfileCache; + private final Pattern queueTopicPattern = Pattern.compile("^[a-zA-Z0-9_.\\-]+$"); + @Override protected void validateCreate(TenantId tenantId, Queue queue) { if (queueDao.findQueueByTenantIdAndName(tenantId, queue.getName()) != null) { @@ -76,8 +80,14 @@ public class QueueValidator extends DataValidator { if (StringUtils.isEmpty(queue.getName())) { throw new DataValidationException("Queue name should be specified!"); } - if (StringUtils.isBlank(queue.getTopic())) { - throw new DataValidationException("Queue topic should be non empty and without spaces!"); + if (!queueTopicPattern.matcher(queue.getName()).matches()) { + throw new DataValidationException("Queue name contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); + } + if (StringUtils.isEmpty(queue.getTopic())) { + throw new DataValidationException("Queue topic should be specified!"); + } + if (!queueTopicPattern.matcher(queue.getTopic()).matches()) { + throw new DataValidationException("Queue topic contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); } if (queue.getPollInterval() < 1) { throw new DataValidationException("Queue poll interval should be more then 0!"); diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java index ce15ec9e88..bfca262abf 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java @@ -34,6 +34,7 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.regex.Pattern; @Component public class TenantProfileDataValidator extends DataValidator { @@ -45,6 +46,8 @@ public class TenantProfileDataValidator extends DataValidator { @Lazy private TenantProfileService tenantProfileService; + private final Pattern queueTopicPattern = Pattern.compile("^[a-zA-Z0-9_.\\-]+$"); + @Override protected void validateDataImpl(TenantId tenantId, TenantProfile tenantProfile) { if (StringUtils.isEmpty(tenantProfile.getName())) { @@ -110,8 +113,14 @@ public class TenantProfileDataValidator extends DataValidator { if (StringUtils.isEmpty(queue.getName())) { throw new DataValidationException("Queue name should be specified!"); } - if (StringUtils.isBlank(queue.getTopic())) { - throw new DataValidationException("Queue topic should be non empty and without spaces!"); + if (!queueTopicPattern.matcher(queue.getName()).matches()) { + throw new DataValidationException("Queue name contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); + } + if (StringUtils.isEmpty(queue.getTopic())) { + throw new DataValidationException("Queue topic should be specified!"); + } + if (!queueTopicPattern.matcher(queue.getTopic()).matches()) { + throw new DataValidationException("Queue topic contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); } if (queue.getPollInterval() < 1) { throw new DataValidationException("Queue poll interval should be more then 0!"); diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html index 6c34e0111f..6e291e9e23 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.html @@ -26,6 +26,9 @@ {{ 'queue.name-unique' | translate }} + + {{ 'queue.name-pattern' | translate }} + diff --git a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts index b84dfc9985..30df2ff7ae 100644 --- a/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts +++ b/ui-ngx/src/app/modules/home/components/queue/queue-form.component.ts @@ -99,7 +99,7 @@ export class QueueFormComponent implements ControlValueAccessor, OnInit, OnDestr ngOnInit() { this.queueFormGroup = this.fb.group( { - name: ['', [Validators.required]], + name: ['', [Validators.required, Validators.pattern(/^[a-zA-Z0-9_.\-]+$/)]], pollInterval: [25, [Validators.min(1), Validators.required]], partitions: [10, [Validators.min(1), Validators.required]], consumerPerPartition: [false, []], diff --git a/ui-ngx/src/assets/locale/locale.constant-en_US.json b/ui-ngx/src/assets/locale/locale.constant-en_US.json index 0286b5f9d2..747a12c336 100644 --- a/ui-ngx/src/assets/locale/locale.constant-en_US.json +++ b/ui-ngx/src/assets/locale/locale.constant-en_US.json @@ -2928,6 +2928,7 @@ "name": "Name", "name-required": "Queue name is required!", "name-unique": "Queue name is not unique!", + "name-pattern": "Queue name contains a character other than ASCII alphanumerics, '.', '_' and '-'!", "queue-required": "Queue is required!", "topic-required": "Queue topic is required!", "poll-interval-required": "Poll interval is required!", From c21a9c116493bf532e5c6bb4616219e1b01c644d Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Tue, 28 Jun 2022 15:58:15 +0200 Subject: [PATCH 8/9] added queue validation tests --- .../dao/service/BaseQueueServiceTest.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/dao/src/test/java/org/thingsboard/server/dao/service/BaseQueueServiceTest.java b/dao/src/test/java/org/thingsboard/server/dao/service/BaseQueueServiceTest.java index 577193ff94..428a21299d 100644 --- a/dao/src/test/java/org/thingsboard/server/dao/service/BaseQueueServiceTest.java +++ b/dao/src/test/java/org/thingsboard/server/dao/service/BaseQueueServiceTest.java @@ -152,6 +152,20 @@ public abstract class BaseQueueServiceTest extends AbstractServiceTest { queueService.saveQueue(queue); } + @Test(expected = DataValidationException.class) + public void testSaveQueueWithInvalidName() { + Queue queue = new Queue(); + queue.setTenantId(tenantId); + queue.setName("Test 1"); + queue.setTopic("tb_rule_engine.test"); + queue.setPollInterval(25); + queue.setPartitions(1); + queue.setPackProcessingTimeout(2000); + queue.setSubmitStrategy(createTestSubmitStrategy()); + queue.setProcessingStrategy(createTestProcessingStrategy()); + queueService.saveQueue(queue); + } + @Test(expected = DataValidationException.class) public void testSaveQueueWithEmptyTopic() { Queue queue = new Queue(); @@ -165,6 +179,20 @@ public abstract class BaseQueueServiceTest extends AbstractServiceTest { queueService.saveQueue(queue); } + @Test(expected = DataValidationException.class) + public void testSaveQueueWithInvalidTopic() { + Queue queue = new Queue(); + queue.setTenantId(tenantId); + queue.setName("Test"); + queue.setTopic("tb rule engine test"); + queue.setPollInterval(25); + queue.setPartitions(1); + queue.setPackProcessingTimeout(2000); + queue.setSubmitStrategy(createTestSubmitStrategy()); + queue.setProcessingStrategy(createTestProcessingStrategy()); + queueService.saveQueue(queue); + } + @Test(expected = DataValidationException.class) public void testSaveQueueWithEmptyPollInterval() { Queue queue = new Queue(); From ac3b133ec0dfdf9c08eac76bb685db4748d18ffb Mon Sep 17 00:00:00 2001 From: YevhenBondarenko Date: Wed, 29 Jun 2022 10:39:33 +0200 Subject: [PATCH 9/9] refactored queue validator --- .../server/dao/service/DataValidator.java | 24 +++++++++++++++++++ .../dao/service/validator/QueueValidator.java | 20 +++------------- .../validator/TenantProfileDataValidator.java | 18 +++----------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java index 1b27a2751f..81297b3594 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/DataValidator.java @@ -17,6 +17,7 @@ package org.thingsboard.server.dao.service; import com.fasterxml.jackson.databind.JsonNode; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.thingsboard.server.common.data.BaseData; import org.thingsboard.server.common.data.EntityType; import org.thingsboard.server.common.data.id.TenantId; @@ -36,6 +37,11 @@ public abstract class DataValidator> { private static final Pattern EMAIL_PATTERN = Pattern.compile("^[A-Z0-9._%+-]+@[A-Z0-9.-]+\\.[A-Z]{2,}$", Pattern.CASE_INSENSITIVE); + private static final Pattern QUEUE_PATTERN = Pattern.compile("^[a-zA-Z0-9_.\\-]+$"); + + private static final String NAME = "name"; + private static final String TOPIC = "topic"; + // Returns old instance of the same object that is fetched during validation. public D validate(D data, Function tenantIdFunction) { try { @@ -134,4 +140,22 @@ public abstract class DataValidator> { } } + protected static void validateQueueName(String name) { + validateQueueNameOrTopic(name, NAME); + } + + protected static void validateQueueTopic(String topic) { + validateQueueNameOrTopic(topic, TOPIC); + } + + private static void validateQueueNameOrTopic(String value, String fieldName) { + if (StringUtils.isEmpty(value)) { + throw new DataValidationException(String.format("Queue %s should be specified!", fieldName)); + } + if (!QUEUE_PATTERN.matcher(value).matches()) { + throw new DataValidationException( + String.format("Queue %s contains a character other than ASCII alphanumerics, '.', '_' and '-'!", fieldName)); + } + } + } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java index a1d0df43d5..1eaa64f446 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/QueueValidator.java @@ -15,7 +15,6 @@ */ package org.thingsboard.server.dao.service.validator; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.thingsboard.server.common.data.TenantProfile; @@ -29,8 +28,6 @@ import org.thingsboard.server.dao.queue.QueueDao; import org.thingsboard.server.dao.service.DataValidator; import org.thingsboard.server.dao.tenant.TbTenantProfileCache; -import java.util.regex.Pattern; - @Component public class QueueValidator extends DataValidator { @@ -40,8 +37,6 @@ public class QueueValidator extends DataValidator { @Autowired private TbTenantProfileCache tenantProfileCache; - private final Pattern queueTopicPattern = Pattern.compile("^[a-zA-Z0-9_.\\-]+$"); - @Override protected void validateCreate(TenantId tenantId, Queue queue) { if (queueDao.findQueueByTenantIdAndName(tenantId, queue.getName()) != null) { @@ -77,18 +72,9 @@ public class QueueValidator extends DataValidator { } } - if (StringUtils.isEmpty(queue.getName())) { - throw new DataValidationException("Queue name should be specified!"); - } - if (!queueTopicPattern.matcher(queue.getName()).matches()) { - throw new DataValidationException("Queue name contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); - } - if (StringUtils.isEmpty(queue.getTopic())) { - throw new DataValidationException("Queue topic should be specified!"); - } - if (!queueTopicPattern.matcher(queue.getTopic()).matches()) { - throw new DataValidationException("Queue topic contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); - } + validateQueueName(queue.getName()); + validateQueueTopic(queue.getTopic()); + if (queue.getPollInterval() < 1) { throw new DataValidationException("Queue poll interval should be more then 0!"); } diff --git a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java index bfca262abf..feab5078e6 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java +++ b/dao/src/main/java/org/thingsboard/server/dao/service/validator/TenantProfileDataValidator.java @@ -34,7 +34,6 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.regex.Pattern; @Component public class TenantProfileDataValidator extends DataValidator { @@ -46,8 +45,6 @@ public class TenantProfileDataValidator extends DataValidator { @Lazy private TenantProfileService tenantProfileService; - private final Pattern queueTopicPattern = Pattern.compile("^[a-zA-Z0-9_.\\-]+$"); - @Override protected void validateDataImpl(TenantId tenantId, TenantProfile tenantProfile) { if (StringUtils.isEmpty(tenantProfile.getName())) { @@ -110,18 +107,9 @@ public class TenantProfileDataValidator extends DataValidator { } private void validateQueueConfiguration(TenantProfileQueueConfiguration queue) { - if (StringUtils.isEmpty(queue.getName())) { - throw new DataValidationException("Queue name should be specified!"); - } - if (!queueTopicPattern.matcher(queue.getName()).matches()) { - throw new DataValidationException("Queue name contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); - } - if (StringUtils.isEmpty(queue.getTopic())) { - throw new DataValidationException("Queue topic should be specified!"); - } - if (!queueTopicPattern.matcher(queue.getTopic()).matches()) { - throw new DataValidationException("Queue topic contains a character other than ASCII alphanumerics, '.', '_' and '-'!"); - } + validateQueueName(queue.getName()); + validateQueueTopic(queue.getTopic()); + if (queue.getPollInterval() < 1) { throw new DataValidationException("Queue poll interval should be more then 0!"); }